diff --git a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst index 414f8a2012d6..cf0f9cdca7e8 100644 --- a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst +++ b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst @@ -410,7 +410,7 @@ workqueues (see Documentation/core-api/workqueue.rst). The requesting task still does counter snapshotting and funnel-lock processing, but the task reaching the top of the funnel lock does a -``schedule_work()`` (from ``_synchronize_rcu_expedited()`` so that a +``schedule_work()`` (from ``_synchronize_rcu_expedited()``) so that a workqueue kthread does the actual grace-period processing. Because workqueue kthreads do not accept POSIX signals, grace-period-wait processing need not allow for POSIX signals. In addition, this approach diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg index d05bc7b27edb..95a66de40ca5 100644 --- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg +++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg @@ -3933,7 +3933,7 @@ font-style="normal" y="-3914.085" x="3745.7725" - xml:space="preserve">rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() a = 42; /* Each field in its own cache line. */ @@ -293,7 +293,7 @@ Then one approach is to use locking, for example, as follows:: { struct foo *p; - p = kmalloc(...); + p = kmalloc_obj(*p); if (p == NULL) deal_with_it(); spin_lock(&p->lock); diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index c13b25544606..96a0681005e0 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6140,6 +6140,11 @@ Kernel parameters the number of CPUs. For example, -2 selects N (the number of CPUs), -3 selects N+1, and so on. + rcutorture.nwriters= [KNL] + Set number of RCU writers, which must be either + zero or one. For additional writers, use instead + the rcutorture.nfakewriters parameter. + rcutorture.object_debug= [KNL] Enable debug-object double-call_rcu() testing. @@ -6229,6 +6234,15 @@ Kernel parameters and stall_gp_kthread are specified, the kthread is starved first, then the CPU. + rcutorture.stall_only= [KNL] + Shut off all rcutorture kthreads other than the + RCU CPU stall-warning test kthreads. The purpose + of this is to test production applictions' + reactions to CPU stalls, and with minimal + additional overhead. Or you can omit the + stall-warning tests as well and get a heavy + no-op, your choice! + rcutorture.stat_interval= [KNL] Time (s) between statistics printk()s. diff --git a/MAINTAINERS b/MAINTAINERS index ac485492481e..144a5d64494a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -25046,6 +25046,7 @@ SLEEPABLE READ-COPY UPDATE (SRCU) M: Lai Jiangshan M: "Paul E. McKenney" M: Josh Triplett +M: Onur Özkan (RUST) R: Steven Rostedt R: Mathieu Desnoyers L: rcu@vger.kernel.org @@ -25054,6 +25055,8 @@ W: http://www.rdrop.com/users/paulmck/RCU/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux.git rcu/dev F: include/linux/srcu*.h F: kernel/rcu/srcu*.c +F: rust/helpers/srcu.c +F: rust/kernel/sync/srcu.rs SMACK SECURITY MODULE M: Casey Schaufler diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 2fdc2208f1ca..08b63ecf719b 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -50,12 +50,14 @@ struct rcu_cblist { * Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also * empty. * - * The ->gp_seq[] array contains the grace-period number at which the - * corresponding segment of callbacks will be ready to invoke. A given - * element of this array is meaningful only when the corresponding segment - * is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks - * are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have - * not yet been assigned a grace-period number). + * The ->gp_seq[] array contains the grace-period state at which the + * corresponding segment of callbacks will be ready to invoke. This tracks + * both normal and expedited grace periods, allowing callbacks to complete + * when either type of GP finishes. A given element of this array is + * meaningful only when the corresponding segment is non-empty, and it is + * never valid for RCU_DONE_TAIL (whose callbacks are already ready to + * invoke) or for RCU_NEXT_TAIL (whose callbacks have not yet been assigned + * a grace-period state). */ #define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */ @@ -190,7 +192,7 @@ struct rcu_cblist { struct rcu_segcblist { struct rcu_head *head; struct rcu_head **tails[RCU_CBLIST_NSEGS]; - unsigned long gp_seq[RCU_CBLIST_NSEGS]; + struct rcu_gp_seq gp_seq[RCU_CBLIST_NSEGS]; #ifdef CONFIG_RCU_NOCB_CPU atomic_long_t len; #else diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..c77b1e02a93c 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -52,9 +52,18 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void synchronize_rcu(void); -struct rcu_gp_oldstate; +/* + * Grace-period sequence snapshot for the polled RCU APIs: ->norm for the + * normal grace period and ->exp for the expedited one. ->exp is unused by + * Tiny RCU, but is present unconditionally so that a single definition + * serves both Tiny RCU and Tree RCU. + */ +struct rcu_gp_seq { + unsigned long norm; + unsigned long exp; +}; unsigned long get_completed_synchronize_rcu(void); -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp); // Maximum number of unsigned long values corresponding to // not-yet-completed RCU grace periods. @@ -490,12 +499,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..273c59a03251 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,17 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); @@ -198,10 +206,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h index 4c92d4291cce..fa884704a3b7 100644 --- a/include/linux/rcupdate_wait.h +++ b/include/linux/rcupdate_wait.h @@ -18,7 +18,7 @@ struct rcu_synchronize { struct completion completion; /* This is for debugging. */ - struct rcu_gp_oldstate oldstate; + struct rcu_gp_seq oldstate; }; void wakeme_after_rcu(struct rcu_head *head); diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index f519cd680228..e56ded733b1b 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -14,11 +14,7 @@ #include /* for HZ */ -struct rcu_gp_oldstate { - unsigned long rgos_norm; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 @@ -26,31 +22,31 @@ struct rcu_gp_oldstate { * Are the two oldstate values the same? See the Tree RCU version for * docbook header. */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm; + return rgosp1->norm == rgosp2->norm; } unsigned long get_state_synchronize_rcu(void); -static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = get_state_synchronize_rcu(); + gsp->norm = get_state_synchronize_rcu(); } unsigned long start_poll_synchronize_rcu(void); -static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu(); + gsp->norm = start_poll_synchronize_rcu(); } bool poll_state_synchronize_rcu(unsigned long oldstate); -static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - return poll_state_synchronize_rcu(rgosp->rgos_norm); + return poll_state_synchronize_rcu(gsp->norm); } static inline void cond_synchronize_rcu(unsigned long oldstate) @@ -58,9 +54,9 @@ static inline void cond_synchronize_rcu(unsigned long oldstate) might_sleep(); } -static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu(rgosp->rgos_norm); + cond_synchronize_rcu(gsp->norm); } static inline unsigned long start_poll_synchronize_rcu_expedited(void) @@ -68,9 +64,9 @@ static inline unsigned long start_poll_synchronize_rcu_expedited(void) return start_poll_synchronize_rcu(); } -static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu_expedited(); + gsp->norm = start_poll_synchronize_rcu_expedited(); } static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) @@ -78,9 +74,9 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) cond_synchronize_rcu(oldstate); } -static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu_expedited(rgosp->rgos_norm); + cond_synchronize_rcu_expedited(gsp->norm); } extern void rcu_barrier(void); diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9d2d7bd251d4..16a04202888b 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -38,12 +38,7 @@ void synchronize_rcu_expedited(void); void rcu_barrier(void); void rcu_momentary_eqs(void); -struct rcu_gp_oldstate { - unsigned long rgos_norm; - unsigned long rgos_exp; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 @@ -60,29 +55,29 @@ struct rcu_gp_oldstate { * to a list header, allowing those structures to be slightly smaller. * * Note that equality is judged on a bitwise basis, so that an - * @rcu_gp_oldstate structure with an already-completed state in one field + * @rcu_gp_seq structure with an already-completed state in one field * will compare not-equal to a structure with an already-completed state - * in the other field. After all, the @rcu_gp_oldstate structure is opaque + * in the other field. After all, the @rcu_gp_seq structure is opaque * so how did such a situation come to pass in the first place? */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; + return rgosp1->norm == rgosp2->norm && rgosp1->exp == rgosp2->exp; } unsigned long start_poll_synchronize_rcu_expedited(void); -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu_expedited(unsigned long oldstate); -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); unsigned long get_state_synchronize_rcu(void); -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); unsigned long start_poll_synchronize_rcu(void); -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp); bool poll_state_synchronize_rcu(unsigned long oldstate); -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu(unsigned long oldstate); -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp); #ifdef CONFIG_PROVE_RCU void rcu_irq_exit_check_preempt(void); diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..c5ab7df6fe5c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,6 +69,13 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 905b629e8fa3..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -154,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index fd1a9270cb9a..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -374,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 5fbdabe3faea..c84309c38834 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -547,10 +547,11 @@ TRACE_EVENT_RCU(rcu_segcb_stats, ), TP_fast_assign( + int i; __entry->ctx = ctx; memcpy(__entry->seglen, rs->seglen, RCU_CBLIST_NSEGS * sizeof(long)); - memcpy(__entry->gp_seq, rs->gp_seq, RCU_CBLIST_NSEGS * sizeof(unsigned long)); - + for (i = 0; i < RCU_CBLIST_NSEGS; i++) + __entry->gp_seq[i] = rs->gp_seq[i].norm; ), TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) " diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 762299291e09..332df7a7a634 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -140,7 +140,6 @@ config FORCE_TASKS_TRACE_RCU config TASKS_TRACE_RCU bool default n - select IRQ_WORK config TASKS_TRACE_RCU_NO_MB bool "Override RCU Tasks Trace inclusion of read-side memory barriers" diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30ce73d1..39a9f6fa9a7b 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -46,16 +46,25 @@ * the number of pending readers that will use * this inactive index is bounded). * - * RCU polled GP special control value: + * RCU polled GP special control values: * * RCU_GET_STATE_COMPLETED : State value indicating an already-completed * polled GP has completed. This value covers * both the state and the counter of the * grace-period sequence number. + * + * RCU_GET_STATE_NOT_TRACKED : State value indicating that a GP component + * is not tracked by this subsystem and should + * not be checked. Used by SRCU and RCU Tasks + * which do not track expedited GPs, to prevent + * false-positive completion when their + * gp_seq entries are checked via + * poll_state_synchronize_rcu_full(). */ -/* Low-order bit definition for polled grace-period APIs. */ +/* Low-order bit definitions for polled grace-period APIs. */ #define RCU_GET_STATE_COMPLETED 0x1 +#define RCU_GET_STATE_NOT_TRACKED 0x2 /* A complete grace period count */ #define RCU_SEQ_GP (RCU_SEQ_STATE_MASK + 1) @@ -695,4 +704,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */ diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 298a2c573f02..dd770006e7f8 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -12,6 +12,7 @@ #include #include +#include "rcu.h" #include "rcu_segcblist.h" /* Initialize simple callback list. */ @@ -307,13 +308,13 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) /* * Return false if there are no CBs awaiting grace periods, otherwise, - * return true and store the nearest waited-upon grace period into *lp. + * return true and store the nearest waited-upon grace period state into *gsp. */ -bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { if (!rcu_segcblist_pend_cbs(rsclp)) return false; - *lp = rsclp->gp_seq[RCU_WAIT_TAIL]; + *gsp = rsclp->gp_seq[RCU_WAIT_TAIL]; return true; } @@ -463,31 +464,15 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, } /* - * Advance the callbacks in the specified rcu_segcblist structure based - * on the current value passed in for the grace-period counter. + * Clean up and compact the segmented callback list after callbacks have been + * advanced to the RCU_DONE_TAIL segment. The @i parameter is the index of the + * first segment that was NOT advanced (i.e., the segment after the last one + * moved to RCU_DONE_TAIL). This function fixes up tail pointers and compacts + * any gaps left by the moved segments. */ -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) +static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) { - int i, j; - - WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); - if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) - return; - - /* - * Find all callbacks whose ->gp_seq numbers indicate that they - * are ready to invoke, and put them into the RCU_DONE_TAIL segment. - */ - for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { - if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) - break; - WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); - rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); - } - - /* If no callbacks moved, nothing more need be done. */ - if (i == RCU_WAIT_TAIL) - return; + int j; /* Clean up tail pointers that might have been misordered above. */ for (j = RCU_WAIT_TAIL; j < i; j++) @@ -508,6 +493,39 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) } } +/* + * Advance the callbacks in the specified rcu_segcblist structure based + * on the current grace-period state. Checks both normal and expedited + * grace periods, advancing callbacks when either GP type completes. + */ +void rcu_segcblist_advance(struct rcu_segcblist *rsclp) +{ + int i; + + WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); + if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) + return; + + /* + * Find all callbacks whose grace periods have completed (either + * normal or expedited) and put them into the RCU_DONE_TAIL segment. + * We check against the current global GP state, which includes + * proper memory barriers and handles special completion values. + */ + for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { + if (!poll_state_synchronize_rcu_full(&rsclp->gp_seq[i])) + break; + WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); + rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); + } + + /* If no callbacks moved, nothing more need be done. */ + if (i == RCU_WAIT_TAIL) + return; + + rcu_segcblist_advance_compact(rsclp, i); +} + /* * "Accelerate" callbacks based on more-accurate grace-period information. * The reason for this is that RCU does not synchronize the beginnings and @@ -519,11 +537,11 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) * them to complete at the end of the earlier grace period. * * This function operates on an rcu_segcblist structure, and also the - * grace-period sequence number seq at which new callbacks would become + * grace-period state gsp at which new callbacks would become * ready to invoke. Returns true if there are callbacks that won't be - * ready to invoke until seq, false otherwise. + * ready to invoke until the grace period represented by gsp, false otherwise. */ -bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) +bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { int i, j; @@ -533,20 +551,20 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) /* * Find the segment preceding the oldest segment of callbacks - * whose ->gp_seq[] completion is at or after that passed in via - * "seq", skipping any empty segments. This oldest segment, along + * whose grace period completion is at or after that passed in via + * "gsp", skipping any empty segments. This oldest segment, along * with any later segments, can be merged in with any newly arrived - * callbacks in the RCU_NEXT_TAIL segment, and assigned "seq" - * as their ->gp_seq[] grace-period completion sequence number. + * callbacks in the RCU_NEXT_TAIL segment, and assigned "gsp" + * as their grace-period completion state. */ for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--) if (!rcu_segcblist_segempty(rsclp, i) && - ULONG_CMP_LT(rsclp->gp_seq[i], seq)) + ULONG_CMP_LT(rsclp->gp_seq[i].norm, gsp->norm)) break; /* * If all the segments contain callbacks that correspond to - * earlier grace-period sequence numbers than "seq", leave. + * earlier grace-period sequence numbers than "gsp", leave. * Assuming that the rcu_segcblist structure has enough * segments in its arrays, this can only happen if some of * the non-done segments contain callbacks that really are @@ -554,15 +572,15 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) * out by the next call to rcu_segcblist_advance(). * * Also advance to the oldest segment of callbacks whose - * ->gp_seq[] completion is at or after that passed in via "seq", + * ->gp_seq[] completion is at or after that passed in via "gsp", * skipping any empty segments. * * Note that segment "i" (and any lower-numbered segments * containing older callbacks) will be unaffected, and their - * grace-period numbers remain unchanged. For example, if i == + * grace-period states remain unchanged. For example, if i == * WAIT_TAIL, then neither WAIT_TAIL nor DONE_TAIL will be touched. * Instead, the CBs in NEXT_TAIL will be merged with those in - * NEXT_READY_TAIL and the grace-period number of NEXT_READY_TAIL + * NEXT_READY_TAIL and the grace-period state of NEXT_READY_TAIL * would be updated. NEXT_TAIL would then be empty. */ if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL) @@ -574,14 +592,14 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) /* * Merge all later callbacks, including newly arrived callbacks, - * into the segment located by the for-loop above. Assign "seq" - * as the ->gp_seq[] value in order to correctly handle the case + * into the segment located by the for-loop above. Assign "gsp" + * as the grace-period state in order to correctly handle the case * where there were no pending callbacks in the rcu_segcblist * structure other than in the RCU_NEXT_TAIL segment. */ for (; i < RCU_NEXT_TAIL; i++) { WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]); - rsclp->gp_seq[i] = seq; + rsclp->gp_seq[i] = *gsp; } return true; } @@ -620,3 +638,42 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, rcu_segcblist_init(src_rsclp); } + +void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) +{ + int i; + + WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); + if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) + return; + + /* + * Find all callbacks whose normal GP sequence numbers indicate + * that they are ready to invoke. For SRCU, we only check norm. + */ + for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { + if (ULONG_CMP_LT(seq, rsclp->gp_seq[i].norm)) + break; + WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); + rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); + } + + /* If no callbacks moved, nothing more need be done. */ + if (i == RCU_WAIT_TAIL) + return; + + rcu_segcblist_advance_compact(rsclp, i); +} + +/* + * SRCU wrapper for rcu_segcblist_accelerate() - converts SRCU's unsigned + * long GP sequence to rcu_gp_seq format with exp set to + * RCU_GET_STATE_NOT_TRACKED (since SRCU does not use expedited GPs) + * and calls the core rcu_segcblist_accelerate(). + */ +bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) +{ + struct rcu_gp_seq gs = { .norm = seq, .exp = RCU_GET_STATE_NOT_TRACKED }; + + return rcu_segcblist_accelerate(rsclp, &gs); +} diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index fadc08ad4b7b..431c4466b889 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -124,7 +124,7 @@ bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); -bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, struct rcu_head *rhp); bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, @@ -139,7 +139,9 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); -bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); +void rcu_segcblist_advance(struct rcu_segcblist *rsclp); +bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, struct rcu_segcblist *src_rsclp); +void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); +bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index ac0b1c6b7dae..1097ec15879c 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -91,6 +91,8 @@ torture_param(int, shutdown_secs, !IS_MODULE(CONFIG_RCU_SCALE_TEST) * 300, torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable"); +torture_param(int, nexp, 0, "Number of expedited GP threads to run concurrently"); +torture_param(int, exp_interval, 0, "Interval (us) between expedited GPs, zero to disable"); torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); @@ -115,8 +117,10 @@ struct writer_freelist { static int nrealreaders; static int nrealwriters; +static int nrealexp; static struct task_struct **writer_tasks; static struct task_struct **reader_tasks; +static struct task_struct **exp_tasks; static u64 **writer_durations; static bool *writer_done; @@ -462,6 +466,34 @@ rcu_scale_reader(void *arg) return 0; } +/* + * RCU expedited GP kthread. Repeatedly invokes expedited grace periods + * to generate concurrent expedited GP load while the normal-GP writers + * are being measured. This allows measuring the benefit of callbacks + * that can piggyback on expedited grace periods. + */ +static int +rcu_scale_exp(void *arg) +{ + long me = (long)arg; + + VERBOSE_SCALEOUT_STRING("rcu_scale_exp task started"); + set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); + set_user_nice(current, MIN_NICE); + + if (holdoff) + schedule_timeout_idle(holdoff * HZ); + + do { + if (exp_interval) + udelay(exp_interval); + cur_ops->exp_sync(); + rcu_scale_wait_shutdown(); + } while (!torture_must_stop()); + torture_kthread_stopping("rcu_scale_exp"); + return 0; +} + /* * Allocate a writer_mblock structure for the specified rcu_scale_writer * task. @@ -664,8 +696,10 @@ static void rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) { pr_alert("%s" SCALE_FLAG - "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown_secs=%d\n", - scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, minruntime, nrealreaders, nrealwriters, writer_holdoff, writer_holdoff_jiffies, verbose, shutdown_secs); + "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d nexp=%d exp_interval=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown_secs=%d\n", + scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, + minruntime, nrealreaders, nrealwriters, nrealexp, exp_interval, + writer_holdoff, writer_holdoff_jiffies, verbose, shutdown_secs); } /* @@ -809,6 +843,13 @@ kfree_scale_cleanup(void) if (torture_cleanup_begin()) return; + if (exp_tasks) { + for (i = 0; i < nrealexp; i++) + torture_stop_kthread(rcu_scale_exp, exp_tasks[i]); + kfree(exp_tasks); + exp_tasks = NULL; + } + if (kfree_reader_tasks) { for (i = 0; i < kfree_nrealthreads; i++) torture_stop_kthread(kfree_scale_thread, @@ -903,6 +944,22 @@ kfree_scale_init(void) goto unwind; } + if (nrealexp > 0 && cur_ops->exp_sync) { + exp_tasks = kzalloc_objs(exp_tasks[0], nrealexp); + if (!exp_tasks) { + SCALEOUT_ERRSTRING("out of memory"); + firsterr = -ENOMEM; + goto unwind; + } + for (i = 0; i < nrealexp; i++) { + firsterr = torture_create_kthread(rcu_scale_exp, + (void *)i, + exp_tasks[i]); + if (torture_init_error(firsterr)) + goto unwind; + } + } + while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads) schedule_timeout_uninterruptible(1); @@ -959,6 +1016,13 @@ rcu_scale_cleanup(void) return; } + if (exp_tasks) { + for (i = 0; i < nrealexp; i++) + torture_stop_kthread(rcu_scale_exp, exp_tasks[i]); + kfree(exp_tasks); + exp_tasks = NULL; + } + if (reader_tasks) { for (i = 0; i < nrealreaders; i++) torture_stop_kthread(rcu_scale_reader, @@ -1076,6 +1140,7 @@ rcu_scale_init(void) if (kthread_tp) kthread_stime = kthread_tp->stime; } + nrealexp = nexp; if (kfree_rcu_test) return kfree_scale_init(); @@ -1107,6 +1172,21 @@ rcu_scale_init(void) } while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders) schedule_timeout_uninterruptible(1); + if (nrealexp > 0 && cur_ops->exp_sync) { + exp_tasks = kzalloc_objs(exp_tasks[0], nrealexp); + if (!exp_tasks) { + SCALEOUT_ERRSTRING("out of memory"); + firsterr = -ENOMEM; + goto unwind; + } + for (i = 0; i < nrealexp; i++) { + firsterr = torture_create_kthread(rcu_scale_exp, + (void *)i, + exp_tasks[i]); + if (torture_init_error(firsterr)) + goto unwind; + } + } writer_tasks = kzalloc_objs(writer_tasks[0], nrealwriters); writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations), GFP_KERNEL); writer_n_durations = kzalloc_objs(*writer_n_durations, nrealwriters); diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 882a158ada7b..794937e13e7c 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -80,6 +80,7 @@ MODULE_AUTHOR("Paul E. McKenney and Josh Triplett get_gp_seq(); ts = rcu_trace_clock_local(); if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) @@ -635,6 +642,7 @@ static struct rcu_torture_ops rcu_ops = { .format_gp_seqs = rcutorture_format_gp_seqs, .set_gpwrap_lag = rcu_set_gpwrap_lag, .get_gpwrap_count = rcu_get_gpwrap_count, + .is_task_rcu_boosted = rcu_is_task_rcu_boosted, .irq_capable = 1, .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), .extendables = RCUTORTURE_MAX_EXTEND, @@ -760,11 +768,16 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) const long uspertick = 1000000 / HZ; const long longdelay = 10; - /* We want there to be long-running readers, but not all the time. */ + // If there is a forward-progress test in flight, don't delay. + if (atomic_read(&rcu_fwd_cb_nodelay)) + return; + + // We want there to be long-running readers, but not all the time. + // The !rcu_preempt_depth() is for RCU Tasks Trace. delay = torture_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); - if (!delay && in_task()) { + if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled()) { schedule_timeout_interruptible(longdelay); rtrsp->rt_delay_jiffies = longdelay; } else { @@ -1213,15 +1226,24 @@ static struct rcu_torture_ops tasks_rude_ops = { * Definitions for tracing RCU-tasks torture testing. */ +// Note that an RCU Tasks Trace GP must imply an RCU GP. static int tasks_tracing_torture_read_lock(void) { - rcu_read_lock_trace(); - return 0; + int use_rcu = !(jiffies & 0xff); + + if (use_rcu) + rcu_read_lock(); + else + rcu_read_lock_trace(); + return use_rcu; } -static void tasks_tracing_torture_read_unlock(int idx) +static void tasks_tracing_torture_read_unlock(int use_rcu) { - rcu_read_unlock_trace(); + if (use_rcu) + rcu_read_unlock(); + else + rcu_read_unlock_trace(); } static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) @@ -1236,7 +1258,8 @@ static struct rcu_torture_ops tasks_tracing_ops = { .read_delay = srcu_read_delay, /* just reuse srcu's version. */ .readunlock = tasks_tracing_torture_read_unlock, .readlock_held = rcu_read_lock_trace_held, - .get_gp_seq = rcu_no_completed, + .get_gp_seq = rcu_tasks_trace_batches_completed, + .gp_diff = rcu_seq_diff, .deferred_free = rcu_tasks_tracing_torture_deferred_free, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, @@ -1605,7 +1628,7 @@ static void rcu_torture_write_types(void) static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) { unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; bool dopoll; bool dopoll_full; unsigned long r = torture_random(trsp); @@ -1653,18 +1676,18 @@ rcu_torture_writer(void *arg) bool booting_still = false; bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; int expediting = 0; unsigned long gp_snap; unsigned long gp_snap1; - struct rcu_gp_oldstate gp_snap_full; - struct rcu_gp_oldstate gp_snap1_full; + struct rcu_gp_seq gp_snap_full; + struct rcu_gp_seq gp_snap1_full; int i; int idx; unsigned long j; struct work_struct lazy_work; int oldnice = task_nice(current); - struct rcu_gp_oldstate *rgo = NULL; + struct rcu_gp_seq *rgo = NULL; int rgo_size = 0; struct rcu_torture *rp; struct rcu_torture *old_rp; @@ -1963,7 +1986,7 @@ static int rcu_torture_fakewriter(void *arg) { unsigned long gp_snap; - struct rcu_gp_oldstate gp_snap_full; + struct rcu_gp_seq gp_snap_full; DEFINE_TORTURE_RANDOM(rand); VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); @@ -2124,6 +2147,8 @@ static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. } +static DEFINE_PER_CPU(bool, torture_in_scf_handler); + // Verify the specified RCUTORTURE_RDR* state. #define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count() static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) @@ -2133,7 +2158,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE) || in_nmi()) return; - WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq(), ROEC_ARGS); + WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq() && !this_cpu_read(torture_in_scf_handler), ROEC_ARGS); WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS); // If CONFIG_PREEMPT_COUNT=n, further checks are unreliable. @@ -2150,7 +2175,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) // Interrupt handlers have all sorts of stuff disabled, so ignore // unintended disabling. - if (in_serving_softirq() || in_hardirq()) + if (in_serving_softirq() || in_hardirq() || this_cpu_read(torture_in_scf_handler)) return; WARN_ONCE(cur_ops->extendables && @@ -2342,12 +2367,19 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) mask |= RCUTORTURE_RDR_RCU_1; } + /* + * Don't mess with interrupt masking in interrupt handlers. + */ + if (in_hardirq() || this_cpu_read(torture_in_scf_handler)) + mask &= ~(preempts_irq | bhs); + /* * Can't enable bh w/irq disabled. */ if (mask & RCUTORTURE_RDR_IRQ) mask |= oldmask & bhs; + /* * Ideally these sequences would be detected in debug builds * (regardless of RT), but until then don't stop testing @@ -2392,7 +2424,7 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, struct struct rcu_torture_one_read_state { bool checkpolling; unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; unsigned long started; struct rcu_torture *p; int readstate; @@ -2401,6 +2433,80 @@ struct rcu_torture_one_read_state { unsigned long long ts; }; +static void rcu_torture_dump_read_segs(struct rt_read_seg *rrsp, int nsegs) +{ + bool firsttime; + int i; + int j; + + firsttime = 1; + for (i = 0; i < nsegs; i++) { + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) + pr_alert("\t%lluus ", div64_u64(rrsp[i].rt_ts, 1000ULL)); + else + pr_alert("\t"); + pr_cont("%d: %#4x", i, rrsp[i].rt_readstate); + if (rrsp[i].rt_delay_jiffies != 0) { + pr_cont("%s%ldjiffies", firsttime ? "" : "+", + rrsp[i].rt_delay_jiffies); + firsttime = 0; + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { + pr_cont(" CPU %2d", rrsp[i].rt_cpu); + if (rrsp[i].rt_cpu != rrsp[i].rt_end_cpu) + pr_cont("->%-2d", rrsp[i].rt_end_cpu); + else + pr_cont(" ..."); + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && + cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { + char buf1[20+1]; + char buf2[20+1]; + char sepchar = '-'; + + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq, buf1, ARRAY_SIZE(buf1)); + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq_end, buf2, ARRAY_SIZE(buf2)); + if (rrsp[i].rt_gp_seq == rrsp[i].rt_gp_seq_end) { + if (buf2[0]) { + for (j = 0; buf2[j]; j++) + buf2[j] = '.'; + if (j) + buf2[j - 1] = ' '; + } + sepchar = ' '; + } + pr_cont(" %s%c%s", buf1, sepchar, buf2); + } + if (rrsp[i].rt_delay_ms != 0) { + pr_cont(" %s%ldms", firsttime ? "" : "+", rrsp[i].rt_delay_ms); + firsttime = 0; + } + if (rrsp[i].rt_delay_us != 0) { + pr_cont(" %s%ldus", firsttime ? "" : "+", rrsp[i].rt_delay_us); + firsttime = 0; + } + pr_cont("%s", rrsp[i].rt_preempted ? " preempted" : ""); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_BH) + pr_cont(" BH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_IRQ) + pr_cont(" IRQ"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) + pr_cont(" PREEMPT"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RBH) + pr_cont(" RBH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_SCHED) + pr_cont(" SCHED"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_1) + pr_cont(" RCU_1"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_2) + pr_cont(" RCU_2"); + pr_cont("\n"); + + } + if (rt_read_preempted) + pr_alert("\tReader was preempted.\n"); +} + static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp, struct torture_random_state *trsp) { @@ -2465,13 +2571,13 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, rtorsp->ts, rtorsp->started, completed); rcu_ftrace_dump(DUMP_ALL); } - __this_cpu_inc(rcu_torture_count[pipe_count]); + this_cpu_inc(rcu_torture_count[pipe_count]); completed = rcutorture_seq_diff(completed, rtorsp->started); if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(rcu_torture_batch[completed]); + this_cpu_inc(rcu_torture_batch[completed]); preempt_enable(); if (rtorsp->checkpolling) { if (cur_ops->get_gp_state && cur_ops->poll_gp_state) @@ -2514,7 +2620,9 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, */ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) { + static int firsttime = 1; int newstate; + unsigned int nsegs; struct rcu_torture_one_read_state rtors; WARN_ON_ONCE(!rcu_is_watching()); @@ -2526,6 +2634,26 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) return false; rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); rcu_torture_one_read_end(&rtors, trsp); + + // This splat will happen on systems built with CONFIG_IRQ_WORK=n + // and on systems where arch_irq_work_has_interrupt() returns false. + // It might also happen on systems using a short-duration clock + // interrupt instead of a self-IPI (powerpc, s390) or that use + // neither a self-IPI nor a short-duration clock interrupts + // (all architectures using the generic implementation + // of arch_irq_work_raise()). On such systems, RCU cannot + // guarantee to immediately deboost RCU readers when the outermost + // rcu_read_unlock() does not end the full segmented RCU read-side + // critical section. + if (cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && + !in_serving_softirq() && !in_hardirq() && !in_nmi() && + READ_ONCE(firsttime) && xchg(&firsttime, 0)) { + WARN_ON_ONCE(deboost_timeliness_check); + nsegs = rtors.rtrsp - rtors.rtseg; + nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); + pr_alert("Slow-deboost rcutorture reader segments:\n"); + rcu_torture_dump_read_segs(rtors.rtseg, nsegs); + } return true; } @@ -2545,7 +2673,7 @@ static void rcu_torture_timer(struct timer_list *unused) atomic_long_inc(&n_rcu_torture_timers); (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); - /* Test call_rcu() invocation from interrupt handler. */ + /* Test call_rcu() invocation from softirq handler. */ if (cur_ops->call) { struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); @@ -2554,6 +2682,41 @@ static void rcu_torture_timer(struct timer_list *unused) } } +static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_irq_rand); + +/* + * RCU torture reader from timer handler. Dereferences rcu_torture_current, + * incrementing the corresponding element of the pipeline array. The + * counter in the element should never be greater than 1, otherwise, the + * RCU implementation is broken. + * + * Note that on some systems, "interrupts" from idle are direct calls + * rather than interrupts. The torture_in_scf_handler per-CPU variable + * accounts for this case. + */ +static void rcu_torture_irq(void *unused) +{ + WARN_ON_ONCE(in_nmi()); + lockdep_assert_irqs_disabled(); + atomic_long_inc(&n_rcu_torture_irqs); + this_cpu_write(torture_in_scf_handler, true); + (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_irq_rand), -1); + this_cpu_write(torture_in_scf_handler, false); + + // Test call_rcu() invocation from interrupt handler. Interrupts + // will always be disabled here, even in CONFIG_PREEMPT_RT=y kernels. + // The "right" thing to do would be to create a special-purpose + // lockless or raw-spinlock-protected allocator, but in the meantime, + // skip testing call_rcu() from interrupt handlers in kernels built + // with either CONFIG_PREEMPT_RT=y or CONFIG_PROVE_LOCKING=y. + if (cur_ops->call && !IS_ENABLED(CONFIG_PROVE_LOCKING) && !IS_ENABLED(CONFIG_PREEMPT_RT)) { + struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); + + if (rhp) + cur_ops->call(rhp, rcu_torture_timer_cb); + } +} + /* * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The @@ -2563,6 +2726,7 @@ static void rcu_torture_timer(struct timer_list *unused) static int rcu_torture_reader(void *arg) { + unsigned long lastscf = jiffies; unsigned long lastsleep = jiffies; long myid = (long)arg; int mynumonline = myid; @@ -2576,8 +2740,25 @@ rcu_torture_reader(void *arg) tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. do { if (irqreader && cur_ops->irq_capable) { - if (!timer_pending(&t)) + if (!timer_pending(&t)) { + int cpu; + mod_timer(&t, jiffies + 1); + preempt_disable(); + cpu = torture_random(&rand) % nr_cpu_ids; + if (!cpu_online(cpu)) { + cpu = cpumask_next(cpu, cpu_online_mask); + if (cpu >= nr_cpu_ids) + cpu = cpumask_next(-1, cpu_online_mask); + } + // An smp_call_function_single() to self is not an interrupt! + if (cpu != smp_processor_id() && + time_after(jiffies, lastscf + HZ * nrealreaders / 50)) { + smp_call_function_single(cpu, rcu_torture_irq, NULL, 0); + lastscf = jiffies; + } + preempt_enable(); + } } if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) schedule_timeout_interruptible(HZ); @@ -2853,10 +3034,11 @@ rcu_torture_stats_print(void) atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), n_rcu_torture_barrier_error, n_rcu_torture_boost_ktrerror); - pr_cont("rtbf: %ld rtb: %ld nt: %ld ", + pr_cont("rtbf: %ld rtb: %ld nt: %ld ni: %ld ", n_rcu_torture_boost_failure, n_rcu_torture_boosts, - atomic_long_read(&n_rcu_torture_timers)); + atomic_long_read(&n_rcu_torture_timers), + atomic_long_read(&n_rcu_torture_irqs)); if (updownreaders) pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired, nmigrates); torture_onoff_stats(); @@ -2913,10 +3095,10 @@ rcu_torture_stats_print(void) if (cur_ops->get_gp_data) cur_ops->get_gp_data(&flags, &gp_seq); wtp = READ_ONCE(writer_task); - pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", + pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n", rcu_torture_writer_state_getname(), rcu_torture_writer_state, gp_seq, flags, - wtp == NULL ? ~0U : wtp->__state, + wtp == NULL ? '?' : task_state_to_char(wtp), wtp == NULL ? -1 : (int)task_cpu(wtp)); if (!splatted && wtp) { sched_show_task(wtp); @@ -3000,7 +3182,7 @@ static void rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) { pr_alert("%s" TORTURE_FLAG - "--- %s: nreaders=%d nfakewriters=%d " + "--- %s: nreaders=%d nwriters=%d nfakewriters=%d " "stat_interval=%d verbose=%d test_no_idle_hz=%d " "shuffle_interval=%d stutter=%d irqreader=%d " "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " @@ -3015,7 +3197,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) "nocbs_nthreads=%d nocbs_toggle=%d " "test_nmis=%d " "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", - torture_type, tag, nrealreaders, nrealfakewriters, + torture_type, tag, nrealreaders, nwriters, nrealfakewriters, stat_interval, verbose, test_no_idle_hz, shuffle_interval, stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, test_boost, cur_ops->can_boost, @@ -3447,13 +3629,17 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) unsigned long stopat; unsigned long stoppedat; - pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); - if (READ_ONCE(rcu_fwd_emergency_stop)) + if (READ_ONCE(rcu_fwd_emergency_stop)) { + pr_alert("%s: Emergency stop, so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Get out of the way quickly, no GP wait! */ - if (!cur_ops->call) + } + if (!cur_ops->call) { + pr_alert("%s: No ->call(), so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Can't do call_rcu() fwd prog without ->call. */ + } /* Loop continuously posting RCU callbacks. */ + pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); atomic_inc(&rcu_fwd_cb_nodelay); cur_ops->sync(); /* Later readers see above write. */ WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); @@ -4020,9 +4206,7 @@ static int rcu_torture_preempt(void *unused) // Wait for preempt_interval ms with up to 100us fuzz. torture_hrtimeout_ms(preempt_interval, 100, &rand); // Select online CPU. - cpu = cpumask_next(cpu, cpu_online_mask); - if (cpu >= nr_cpu_ids) - cpu = cpumask_next(-1, cpu_online_mask); + cpu = cpumask_next_wrap(cpu, cpu_online_mask); WARN_ON_ONCE(cpu >= nr_cpu_ids); // Move to that CPU, if can't do so, retry later. if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) @@ -4094,11 +4278,9 @@ static void rcu_gpwrap_lag_cleanup(void) static void rcu_torture_cleanup(void) { - int firsttime; int flags = 0; unsigned long gp_seq = 0; int i; - int j; if (torture_cleanup_begin()) { if (cur_ops->cb_barrier != NULL) { @@ -4183,76 +4365,8 @@ rcu_torture_cleanup(void) pr_alert("Failure/close-call rcutorture reader segments:\n"); if (rt_read_nsegs == 0) pr_alert("\t: No segments recorded!!!\n"); - firsttime = 1; - for (i = 0; i < rt_read_nsegs; i++) { - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) - pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL)); - else - pr_alert("\t"); - pr_cont("%d: %#4x", i, err_segs[i].rt_readstate); - if (err_segs[i].rt_delay_jiffies != 0) { - pr_cont("%s%ldjiffies", firsttime ? "" : "+", - err_segs[i].rt_delay_jiffies); - firsttime = 0; - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { - pr_cont(" CPU %2d", err_segs[i].rt_cpu); - if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu) - pr_cont("->%-2d", err_segs[i].rt_end_cpu); - else - pr_cont(" ..."); - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && - cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { - char buf1[20+1]; - char buf2[20+1]; - char sepchar = '-'; - - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq, - buf1, ARRAY_SIZE(buf1)); - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end, - buf2, ARRAY_SIZE(buf2)); - if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) { - if (buf2[0]) { - for (j = 0; buf2[j]; j++) - buf2[j] = '.'; - if (j) - buf2[j - 1] = ' '; - } - sepchar = ' '; - } - pr_cont(" %s%c%s", buf1, sepchar, buf2); - } - if (err_segs[i].rt_delay_ms != 0) { - pr_cont(" %s%ldms", firsttime ? "" : "+", - err_segs[i].rt_delay_ms); - firsttime = 0; - } - if (err_segs[i].rt_delay_us != 0) { - pr_cont(" %s%ldus", firsttime ? "" : "+", - err_segs[i].rt_delay_us); - firsttime = 0; - } - pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : ""); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH) - pr_cont(" BH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ) - pr_cont(" IRQ"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) - pr_cont(" PREEMPT"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH) - pr_cont(" RBH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED) - pr_cont(" SCHED"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1) - pr_cont(" RCU_1"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2) - pr_cont(" RCU_2"); - pr_cont("\n"); - - } - if (rt_read_preempted) - pr_alert("\tReader was preempted.\n"); + else + rcu_torture_dump_read_segs(err_segs, rt_read_nsegs); } if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); @@ -4557,6 +4671,23 @@ rcu_torture_init(void) cur_ops = NULL; goto unwind; } + if (stall_only) { + pr_alert("rcu-torture: stall_only specified, suppressing all else.\n"); + fqs_stutter = 0; + fwd_progress = 0; + n_barrier_cbs = 0; + nfakewriters = 0; + nocbs_nthreads = 0; + nreaders = 0; + n_up_down = 0; + nwriters = 0; + onoff_interval = 0; + preempt_duration = 0; + read_exit_burst = 0; + shuffle_interval = 0; + stutter = 0; + test_boost = 0; + } if (cur_ops->fqs == NULL && fqs_duration != 0) { pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); fqs_duration = 0; @@ -4663,10 +4794,11 @@ rcu_torture_init(void) goto unwind; } - firsterr = torture_create_kthread(rcu_torture_writer, NULL, - writer_task); - if (torture_init_error(firsterr)) - goto unwind; + if (nwriters) { + firsterr = torture_create_kthread(rcu_torture_writer, NULL, writer_task); + if (torture_init_error(firsterr)) + goto unwind; + } firsterr = rcu_torture_updown_init(); if (torture_init_error(firsterr)) diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index a2e2d516e51b..558ba8d316db 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -48,31 +48,31 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, - struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /* - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Must invoke this on a given srcu_struct before passing that srcu_struct * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ @@ -85,7 +85,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); */ void cleanup_srcu_struct(struct srcu_struct *ssp) { - WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); + WARN_ON(srcu_readers_active(ssp)); irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 7c2f7cc131f7..304112674e8a 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -266,12 +266,13 @@ __init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock return init_srcu_struct_fields(ssp, false); } -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { ssp->srcu_reader_flavor = 0; return __init_srcu_struct_common(ssp, name, key); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { @@ -291,7 +292,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Use this in place of DEFINE_SRCU() and DEFINE_STATIC_SRCU() @@ -301,12 +302,12 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = 0; return init_srcu_struct_fields(ssp, false); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); /** * init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure @@ -598,31 +599,6 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks); } -/** - * srcu_readers_active - returns true if there are readers. and false - * otherwise - * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). - * - * Note that this is not an atomic primitive, and can therefore suffer - * severe errors when invoked on an active srcu_struct. That said, it - * can be useful as an error check at cleanup time. - */ -static bool srcu_readers_active(struct srcu_struct *ssp) -{ - int cpu; - unsigned long sum = 0; - - for_each_possible_cpu(cpu) { - struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - - sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); - sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); - sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); - sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); - } - return sum; -} - /* * We use an adaptive strategy for synchronize_srcu() and especially for * synchronize_srcu_expedited(). We spin for a fixed time period @@ -725,7 +701,11 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) for_each_possible_cpu(cpu) { struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - timer_delete_sync(&sdp->delay_work); + // Call srcu_barrier() before this cleanup_srcu_struct() + // to avoid triggering this WARN_ON(). + if (WARN_ON(timer_delete_sync(&sdp->delay_work)) && + rcu_cpu_beenfullyonline(sdp->cpu)) + queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); flush_work(&sdp->work); if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist))) return; /* Forgot srcu_barrier(), so just leak it! */ @@ -1351,7 +1331,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * 2) The grace period for RCU_WAIT_TAIL is seen as started but not * completed so rcu_seq_current() returns X + SRCU_STATE_SCAN1. * - * 3) This value is passed to rcu_segcblist_advance() which can't move + * 3) This value is passed to srcu_segcblist_advance() which can't move * any segment forward and fails. * * 4) srcu_gp_start_if_needed() still proceeds with callback acceleration. @@ -1360,15 +1340,15 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * RCU_NEXT_READY_TAIL segment as started (ie: X + 4 + SRCU_STATE_SCAN1) * so it returns a snapshot of the next grace period, which is X + 12. * - * 5) The value of X + 12 is passed to rcu_segcblist_accelerate() but the + * 5) The value of X + 12 is passed to srcu_segcblist_accelerate() but the * freshly enqueued callback in RCU_NEXT_TAIL can't move to * RCU_NEXT_READY_TAIL which already has callbacks for a previous grace * period (gp_num = X + 8). So acceleration fails. */ s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq); if (rhp) { - rcu_segcblist_advance(&sdp->srcu_cblist, - rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); + srcu_segcblist_advance(&sdp->srcu_cblist, + rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); /* * Acceleration can never fail because the base current gp_seq * used for acceleration is <= the value of gp_seq used for @@ -1376,7 +1356,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * always be able to be emptied by the acceleration into the * RCU_NEXT_READY_TAIL or RCU_WAIT_TAIL segments. */ - WARN_ON_ONCE(!rcu_segcblist_accelerate(&sdp->srcu_cblist, s)); + WARN_ON_ONCE(!srcu_segcblist_accelerate(&sdp->srcu_cblist, s)); } if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { sdp->srcu_gp_seq_needed = s; @@ -1891,8 +1871,8 @@ static void srcu_invoke_callbacks(struct work_struct *work) rcu_cblist_init(&ready_cbs); raw_spin_lock_irq_rcu_node(sdp); WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL)); - rcu_segcblist_advance(&sdp->srcu_cblist, - rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); + srcu_segcblist_advance(&sdp->srcu_cblist, + rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); /* * Although this function is theoretically re-entrant, concurrent * callbacks invocation is disallowed to avoid executing an SRCU barrier diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..627295396cd9 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -67,7 +67,6 @@ struct rcu_tasks_percpu { * @gp_start: Most recent grace-period start in jiffies. * @tasks_gp_seq: Number of grace periods completed since boot in upper bits. * @n_ipis: Number of IPIs sent to encourage grace periods to end. - * @n_ipis_fails: Number of IPI-send failures. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). @@ -102,7 +101,6 @@ struct rcu_tasks { unsigned long gp_start; unsigned long tasks_gp_seq; unsigned long n_ipis; - unsigned long n_ipis_fails; struct task_struct *kthread_ptr; unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; @@ -157,8 +155,8 @@ static struct rcu_tasks rt_name = \ #ifdef CONFIG_TASKS_RCU /* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */ -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused); -static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall); +static void tasks_rcu_exit_stall(struct timer_list *unused); +static DEFINE_TIMER(tasks_rcu_exit_stall_timer, tasks_rcu_exit_stall); #endif /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */ @@ -397,7 +395,11 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } rcu_read_unlock(); - /* We can't create the thread unless interrupts are enabled. */ + // We can't create the kthread with interrupts disabled because a + // scheduler spinlock might be held, so kthread creation is deferred + // until core_initcall() time. Similarly, wakeups are deferred using + // irq_work in order to avoid potential scheduler-lock-deadlock + // lockdep splats. if (needwake && READ_ONCE(rtp->kthread_ptr)) irq_work_queue(&rtpcp->rtp_irq_work); } @@ -481,8 +483,8 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) if (cpu > 0) ncbsnz += n; } - rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); - (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); + srcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); + (void)srcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) { if (rtp->lazy_jiffies) rtpcp->urgent_gp--; @@ -565,7 +567,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu if (rcu_segcblist_empty(&rtpcp->cblist)) return; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); - rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); + srcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); len = rcl.len; @@ -578,7 +580,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu } raw_spin_lock_irqsave_rcu_node(rtpcp, flags); rcu_segcblist_add_len(&rtpcp->cblist, -len); - (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); + (void)srcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } @@ -683,7 +685,6 @@ static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp) t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname); if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name)) return; - smp_mb(); /* Ensure others see full kthread. */ } #ifndef CONFIG_TINY_RCU @@ -722,6 +723,7 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) bool havecbs = false; bool haveurgent = false; bool haveurgentcbs = false; + bool havependtimer = false; for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); @@ -732,19 +734,22 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) haveurgent = true; if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp)) haveurgentcbs = true; - if (havecbs && haveurgent && haveurgentcbs) + if (data_race(timer_pending(&rtpcp->lazy_timer))) + havependtimer = true; + if (havecbs && haveurgent && haveurgentcbs && havependtimer) break; } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c%c%c l:%lu %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), data_race(rcu_seq_current(&rtp->tasks_gp_seq)), - data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), + data_race(rtp->n_ipis), ".k"[!!data_race(rtp->kthread_ptr)], ".C"[havecbs], ".u"[haveurgent], ".U"[haveurgentcbs], + ".P"[havependtimer], rtp->lazy_jiffies, s); } @@ -1027,8 +1032,8 @@ static void rcu_tasks_postscan(struct list_head *hop) int rtsi = READ_ONCE(rcu_task_stall_info); if (!IS_ENABLED(CONFIG_TINY_RCU)) { - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); } /* @@ -1081,7 +1086,7 @@ static void rcu_tasks_postscan(struct list_head *hop) } if (!IS_ENABLED(CONFIG_TINY_RCU)) - timer_delete_sync(&tasks_rcu_exit_srcu_stall_timer); + timer_delete_sync(&tasks_rcu_exit_stall_timer); } /* See if tasks are still holding out, complain if so. */ @@ -1153,7 +1158,7 @@ static void rcu_tasks_postgp(struct rcu_tasks *rtp) synchronize_rcu(); } -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) +static void tasks_rcu_exit_stall(struct timer_list *unused) { #ifndef CONFIG_TINY_RCU int rtsi; @@ -1163,8 +1168,8 @@ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) __func__, rcu_tasks.kname, rcu_tasks.tasks_gp_seq, tasks_gp_state_getname(&rcu_tasks), jiffies - rcu_tasks.gp_jiffies); pr_info("Please check any exiting tasks stuck between calls to exit_tasks_rcu_start() and exit_tasks_rcu_finish()\n"); - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); #endif // #ifndef CONFIG_TINY_RCU } @@ -1174,7 +1179,7 @@ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU + * period elapses, in other words after all currently executing rcu-tasks * read-side critical sections have completed. call_rcu_tasks() assumes * that the read-side critical sections end at a voluntary context * switch (not a preemption!), cond_resched_tasks_rcu_qs(), entry into idle, @@ -1360,8 +1365,8 @@ DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude, * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU - * read-side critical sections have completed. call_rcu_tasks_rude() + * period elapses, in other words after all currently executing rude + * rcu-tasks read-side critical sections have completed. call_rcu_tasks_rude() * assumes that the read-side critical sections end at context switch, * cond_resched_tasks_rcu_qs(), or transition to usermode execution (as * usermode execution is schedulable). As such, there are no read-side @@ -1385,7 +1390,7 @@ static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func) * * Control will return to the caller some time after a rude rcu-tasks * grace period has elapsed, in other words after all currently - * executing rcu-tasks read-side critical sections have elapsed. These + * executing rude rcu-tasks read-side critical sections have elapsed. These * read-side critical sections are delimited by calls to schedule(), * cond_resched_tasks_rcu_qs(), userspace execution (which is a schedulable * context), and (in theory, anyway) cond_resched(). @@ -1455,6 +1460,7 @@ struct rcu_tasks_test_desc { const char *name; bool notrun; unsigned long runstart; + void (*gp_dbg)(void); }; static struct rcu_tasks_test_desc tests[] = { @@ -1462,6 +1468,8 @@ static struct rcu_tasks_test_desc tests[] = { .name = "call_rcu_tasks()", /* If not defined, the test is skipped. */ .notrun = IS_ENABLED(CONFIG_TASKS_RCU), + /* Dump rcu tasks status, if test failed. */ + .gp_dbg = show_rcu_tasks_classic_gp_kthread }, { .name = "call_rcu_tasks_trace()", @@ -1521,6 +1529,8 @@ static int rcu_tasks_verify_self_tests(void) while (tests[i].notrun) { // still hanging. if (time_after(jiffies, tests[i].runstart + bst)) { pr_err("%s has failed boot-time tests.\n", tests[i].name); + if (tests[i].gp_dbg) + tests[i].gp_dbg(); ret = -1; break; } @@ -1606,4 +1616,10 @@ static inline void rcu_tasks_bootup_oddness(void) {} DEFINE_SRCU_FAST(rcu_tasks_trace_srcu_struct); EXPORT_SYMBOL_GPL(rcu_tasks_trace_srcu_struct); +unsigned long rcu_tasks_trace_batches_completed(void) +{ + return srcu_batches_completed(&rcu_tasks_trace_srcu_struct); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_batches_completed); + #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 585cade21010..dccccd6be941 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -187,9 +187,9 @@ EXPORT_SYMBOL_GPL(call_rcu); * Store a grace-period-counter "cookie". For more information, * see the Tree RCU header comment. */ -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; + gsp->norm = RCU_GET_STATE_COMPLETED; } EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 03a43d3d2616..21b6ce1dffb6 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -357,9 +357,10 @@ bool rcu_watching_zero_in_eqs(int cpu, int *vp) */ notrace void rcu_momentary_eqs(void) { + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; - raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); + WRITE_ONCE(rdp->rcu_need_heavy_qs, false); seq = ct_state_inc(2 * CT_RCU_WATCHING); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(seq & CT_RCU_WATCHING)); @@ -986,8 +987,8 @@ static int rcu_watching_snap_recheck(struct rcu_data *rdp) } /* Trace-event wrapper function for trace_rcu_future_grace_period. */ -static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, - unsigned long gp_seq_req, const char *s) +static void trace_rcu_this_gp(struct rcu_node *rnp, unsigned long gp_seq_req, + const char *s) { trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), gp_seq_req, rnp->level, @@ -1026,7 +1027,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, * Note that rnp_start->lock must not be released. */ raw_lockdep_assert_held_rcu_node(rnp_start); - trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf")); + trace_rcu_this_gp(rnp_start, gp_seq_req, TPS("Startleaf")); for (rnp = rnp_start; 1; rnp = rnp->parent) { if (rnp != rnp_start) raw_spin_lock_rcu_node(rnp); @@ -1034,8 +1035,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, rcu_seq_started(&rnp->gp_seq, gp_seq_req) || (rnp != rnp_start && rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) { - trace_rcu_this_gp(rnp, rdp, gp_seq_req, - TPS("Prestarted")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("Prestarted")); goto unlock_out; } WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req); @@ -1046,7 +1046,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, * rcu_gp_cleanup() will see the marking. Bail to * reduce contention. */ - trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, + trace_rcu_this_gp(rnp_start, gp_seq_req, TPS("Startedleaf")); goto unlock_out; } @@ -1058,14 +1058,14 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, /* If GP already in progress, just leave, otherwise start one. */ if (rcu_gp_in_progress()) { - trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("Startedleafroot")); goto unlock_out; } - trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("Startedroot")); WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); WRITE_ONCE(rcu_state.gp_req_activity, jiffies); if (!READ_ONCE(rcu_state.gp_kthread)) { - trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("NoGPkthread")); goto unlock_out; } trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq")); @@ -1088,12 +1088,11 @@ unlock_out: static bool rcu_future_gp_cleanup(struct rcu_node *rnp) { bool needmore; - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed); if (!needmore) rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */ - trace_rcu_this_gp(rnp, rdp, rnp->gp_seq, + trace_rcu_this_gp(rnp, rnp->gp_seq, needmore ? TPS("CleanupMore") : TPS("Cleanup")); return needmore; } @@ -1139,7 +1138,7 @@ static void rcu_gp_kthread_wake(void) */ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) { - unsigned long gp_seq_req; + struct rcu_gp_seq gs; bool ret = false; rcu_lockdep_assert_cblist_protected(rdp); @@ -1161,15 +1160,15 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) * accelerating callback invocation to an earlier grace-period * number. */ - gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); - if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) - ret = rcu_start_this_gp(rnp, rdp, gp_seq_req); + get_state_synchronize_rcu_full(&gs); + if (rcu_segcblist_accelerate(&rdp->cblist, &gs)) + ret = rcu_start_this_gp(rnp, rdp, gs.norm); /* Trace depending on how much we were able to accelerate. */ if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); + trace_rcu_grace_period(rcu_state.name, gs.norm, TPS("AccWaitCB")); else - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); + trace_rcu_grace_period(rcu_state.name, gs.norm, TPS("AccReadyCB")); trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); @@ -1186,14 +1185,14 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, struct rcu_data *rdp) { - unsigned long c; + struct rcu_gp_seq gs; bool needwake; rcu_lockdep_assert_cblist_protected(rdp); - c = rcu_seq_snap(&rcu_state.gp_seq); - if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { + get_state_synchronize_rcu_full(&gs); + if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, gs.norm)) { /* Old request still live, so mark recent callbacks. */ - (void)rcu_segcblist_accelerate(&rdp->cblist, c); + (void)rcu_segcblist_accelerate(&rdp->cblist, &gs); return; } raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ @@ -1206,7 +1205,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, /* * Move any callbacks whose grace period has completed to the * RCU_DONE_TAIL sublist, then compact the remaining sublists and - * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL + * assign ->gp_seq[] state to any callbacks in the RCU_NEXT_TAIL * sublist. This function is idempotent, so it does not hurt to * invoke it repeatedly. As long as it is not invoked -too- often... * Returns true if the RCU grace-period kthread needs to be awakened. @@ -1223,10 +1222,10 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) return false; /* - * Find all callbacks whose ->gp_seq numbers indicate that they - * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. + * Find all callbacks whose grace periods have completed (either + * normal or expedited) and put them into the RCU_DONE_TAIL sublist. */ - rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); + rcu_segcblist_advance(&rdp->cblist); /* Classify any remaining callbacks. */ return rcu_accelerate_cbs(rnp, rdp); @@ -2221,8 +2220,15 @@ static noinline void rcu_gp_cleanup(void) dump_blkd_tasks(rnp, 10); WARN_ON_ONCE(rnp->qsmask); WRITE_ONCE(rnp->gp_seq, new_gp_seq); - if (!rnp->parent) - smp_mb(); // Order against failing poll_state_synchronize_rcu_full(). + if (!rnp->parent) { + /* + * Order against failing poll_state_synchronize_rcu_full(), + * and also against rcu_nocb_gp_cleanup() -> swait_active(), + * which relies on this barrier to observe a waiter that + * enqueued before re-checking the grace-period state. + */ + smp_mb(); + } rdp = this_cpu_ptr(&rcu_data); if (rnp == rdp->mynode) needgp = __note_gp_changes(rnp, rdp) || needgp; @@ -2252,8 +2258,7 @@ static noinline void rcu_gp_cleanup(void) /* Check for GP requests since above loop. */ rdp = this_cpu_ptr(&rcu_data); if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) { - trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed, - TPS("CleanupMore")); + trace_rcu_this_gp(rnp, rnp->gp_seq_needed, TPS("CleanupMore")); needgp = true; } /* Advance CBs to reduce false positives below. */ @@ -2539,7 +2544,7 @@ rcu_check_quiescent_state(struct rcu_data *rdp) * Was there a quiescent state since the beginning of the grace * period? If no, then exit and wait for the next call. */ - if (rdp->cpu_no_qs.b.norm) + if (READ_ONCE(rdp->cpu_no_qs.b.norm)) return; /* @@ -2670,7 +2675,7 @@ static void rcu_do_batch(struct rcu_data *rdp) // reporting, so check time limits for them. if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING && rcu_do_batch_check_time(count, tlimit, jlimit_check, jlimit)) { - rdp->rcu_cpu_has_work = 1; + WRITE_ONCE(rdp->rcu_cpu_has_work, 1); break; } } @@ -2881,6 +2886,23 @@ static __latent_entropy void rcu_core(void) /* Update RCU state based on any recent quiescent states. */ rcu_check_quiescent_state(rdp); + /* Advance callbacks if an expedited GP has completed. */ + if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_is_enabled(&rdp->cblist)) { + struct rcu_gp_seq gp_state; + + if (rcu_segcblist_nextgp(&rdp->cblist, &gp_state) && + poll_state_synchronize_rcu_full(&gp_state)) { + guard(irqsave)(); + if (raw_spin_trylock_rcu_node(rnp)) { + bool needwake = rcu_advance_cbs(rnp, rdp); + + raw_spin_unlock_rcu_node(rnp); + if (needwake) + rcu_gp_kthread_wake(); + } + } + } + /* No grace period and unregistered callbacks? */ if (!rcu_gp_in_progress() && rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp)) { @@ -2889,7 +2911,7 @@ static __latent_entropy void rcu_core(void) rcu_accelerate_cbs_unlocked(rnp, rdp); } - rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); + rcu_check_gp_start_stall(rnp, rcu_jiffies_till_stall_check()); /* If there are callbacks ready, invoke them. */ if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) && @@ -2930,7 +2952,7 @@ static void invoke_rcu_core_kthread(void) unsigned long flags; local_irq_save(flags); - __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); + this_cpu_write(rcu_data.rcu_cpu_has_work, 1); t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); if (t != NULL && t != current) rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); @@ -2957,7 +2979,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu) static int rcu_cpu_kthread_should_run(unsigned int cpu) { - return __this_cpu_read(rcu_data.rcu_cpu_has_work); + return this_cpu_read(rcu_data.rcu_cpu_has_work); } /* @@ -2978,7 +3000,7 @@ static void rcu_cpu_kthread(unsigned int cpu) local_bh_disable(); *statusp = RCU_KTHREAD_RUNNING; local_irq_disable(); - work = *workp; + work = READ_ONCE(*workp); WRITE_ONCE(*workp, 0); local_irq_enable(); if (work) @@ -3023,7 +3045,7 @@ static int __init rcu_spawn_core_kthreads(void) return 0; } -static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func) +static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head) { rcu_segcblist_enqueue(&rdp->cblist, head); trace_rcu_callback(rcu_state.name, head, @@ -3035,9 +3057,9 @@ static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_cal * Handle any core-RCU processing required by a call_rcu() invocation. */ static void call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags) + unsigned long flags) { - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); /* * If called from an extended quiescent state, invoke the RCU * core in order to force a re-evaluation of RCU's idleness. @@ -3178,9 +3200,9 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) check_cb_ovld(rdp); if (unlikely(rcu_rdp_is_offloaded(rdp))) - call_rcu_nocb(rdp, head, func, flags, lazy); + call_rcu_nocb(rdp, head, flags, lazy); else - call_rcu_core(rdp, head, func, flags); + call_rcu_core(rdp, head, flags); local_irq_restore(flags); } @@ -3287,7 +3309,7 @@ EXPORT_SYMBOL_GPL(call_rcu); * Later on, this could in theory be the case for kernels built with * CONFIG_SMP=y && CONFIG_PREEMPTION=y running on a single CPU, but this * is not a common case. Furthermore, this optimization would cause - * the rcu_gp_oldstate structure to expand by 50%, so this potential + * the rcu_gp_seq structure to expand by 50%, so this potential * grace-period optimization is ignored once the scheduler is running. */ static int rcu_blocking_is_gp(void) @@ -3416,16 +3438,16 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); /** * get_completed_synchronize_rcu_full - Return a full pre-completed polled state cookie - * @rgosp: Place to put state cookie + * @gsp: Place to put state cookie * - * Stores into @rgosp a value that will always be treated by functions + * Stores into @gsp a value that will always be treated by functions * like poll_state_synchronize_rcu_full() as a cookie whose grace period * has already completed. */ -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; - rgosp->rgos_exp = RCU_GET_STATE_COMPLETED; + gsp->norm = RCU_GET_STATE_COMPLETED; + gsp->exp = RCU_GET_STATE_COMPLETED; } EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); @@ -3449,13 +3471,13 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); /** * get_state_synchronize_rcu_full - Snapshot RCU state, both normal and expedited - * @rgosp: location to place combined normal/expedited grace-period state + * @gsp: location to place combined normal/expedited grace-period state * - * Places the normal and expedited grace-period states in @rgosp. This + * Places the normal and expedited grace-period states in @gsp. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. - * The rcu_gp_oldstate structure takes up twice the memory of an unsigned + * The rcu_gp_seq structure takes up twice the memory of an unsigned * long, but is guaranteed to see all grace periods. In contrast, the * combined state occupies less memory, but can sometimes fail to take * grace periods into account. @@ -3463,7 +3485,7 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); * This does not guarantee that the needed grace period will actually * start. */ -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { /* * Any prior manipulation of RCU-protected data must happen @@ -3475,8 +3497,8 @@ void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) // in poll_state_synchronize_rcu_full() notwithstanding. Use of // the latter here would result in too-short grace periods due to // interactions with newly onlined CPUs. - rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq); - rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence); + gsp->norm = rcu_seq_snap(&rcu_state.gp_seq); + gsp->exp = rcu_seq_snap(&rcu_state.expedited_sequence); } EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full); @@ -3527,18 +3549,18 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); /** * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * @gsp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * - * Places the normal and expedited grace-period states in *@rgos. This + * Places the normal and expedited grace-period states in *@gs. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. * If the needed grace period is not already slated to start, notifies * RCU core of the need for that grace period. */ -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - get_state_synchronize_rcu_full(rgosp); + get_state_synchronize_rcu_full(gsp); start_poll_synchronize_rcu_common(); } @@ -3588,21 +3610,39 @@ bool poll_state_synchronize_rcu(unsigned long oldstate) } EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); +/* + * Racy, memory-ordering-free test of whether the normal or expedited grace + * period recorded in *gsp has completed. Callers that need the full + * memory-ordering guarantees must use poll_state_synchronize_rcu_full(); + * this variant is only a hint (e.g. for rcu_pending()) and leaves any + * required ordering to a subsequent ordered check. + */ +static bool poll_state_synchronize_rcu_full_unordered(struct rcu_gp_seq *gsp) +{ + struct rcu_node *rnp = rcu_get_root(); + + return gsp->norm == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rnp->gp_seq, gsp->norm) || + gsp->exp == RCU_GET_STATE_COMPLETED || + (gsp->exp != RCU_GET_STATE_NOT_TRACKED && + rcu_seq_done_exact(&rcu_state.expedited_sequence, gsp->exp)); +} + /** * poll_state_synchronize_rcu_full - Has the specified RCU grace period completed? - * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * @gsp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * * If a full RCU grace period has elapsed since the earlier call from - * which *rgosp was obtained, return @true, otherwise return @false. + * which *gsp was obtained, return @true, otherwise return @false. * If @false is returned, it is the caller's responsibility to invoke this * function later on until it does return @true. Alternatively, the caller - * can explicitly wait for a grace period, for example, by passing @rgosp + * can explicitly wait for a grace period, for example, by passing @gsp * to cond_synchronize_rcu() or by directly invoking synchronize_rcu(). * * Yes, this function does not take counter wrap into account. * But counter wrap is harmless. If the counter wraps, we have waited * for more than a billion grace periods (and way more on a 64-bit - * system!). Those needing to keep rcu_gp_oldstate values for very + * system!). Those needing to keep rcu_gp_seq values for very * long time periods (many hours even on 32-bit systems) should check * them occasionally and either refresh them or set a flag indicating * that the grace period has completed. Alternatively, they can use @@ -3611,7 +3651,7 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); * * This function provides the same memory-ordering guarantees that would * be provided by a synchronize_rcu() that was invoked at the call to - * the function that provided @rgosp, and that returned at the end of this + * the function that provided @gsp, and that returned at the end of this * function. And this guarantee requires that the root rcu_node structure's * ->gp_seq field be checked instead of that of the rcu_state structure. * The problem is that the just-ending grace-period's callbacks can be @@ -3621,15 +3661,10 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); * cause a subsequent poll_state_synchronize_rcu_full() to return @true, * then the root rcu_node structure is the one that needs to be polled. */ -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - struct rcu_node *rnp = rcu_get_root(); - smp_mb(); // Order against root rcu_node structure grace-period cleanup. - if (rgosp->rgos_norm == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) || - rgosp->rgos_exp == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) { + if (poll_state_synchronize_rcu_full_unordered(gsp)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } @@ -3664,11 +3699,11 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu); /** * cond_synchronize_rcu_full - Conditionally wait for an RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * @gsp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() * * If a full RCU grace period has elapsed since the call to * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), - * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * or start_poll_synchronize_rcu_expedited_full() from which @gsp was * obtained, just return. Otherwise, invoke synchronize_rcu() to wait * for a full grace period. * @@ -3679,12 +3714,12 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu); * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @rgosp and that returned at the end of + * to the function that provided @gsp and that returned at the end of * this function. */ -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - if (!poll_state_synchronize_rcu_full(rgosp)) + if (!poll_state_synchronize_rcu_full(gsp)) synchronize_rcu(); } EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full); @@ -3699,6 +3734,7 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full); static int rcu_pending(int user) { bool gp_in_progress; + struct rcu_gp_seq gp_state; struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct rcu_node *rnp = rdp->mynode; @@ -3729,6 +3765,17 @@ static int rcu_pending(int user) rcu_segcblist_ready_cbs(&rdp->cblist)) return 1; + /* + * Has a GP (normal or expedited) completed for pending callbacks? + * This is only a racy hint to decide whether to run rcu_core(); the + * ordered re-check and callback advancement happen there, so the + * unordered test avoids paying for memory barriers on every tick. + */ + if (!rcu_rdp_is_offloaded(rdp) && + rcu_segcblist_nextgp(&rdp->cblist, &gp_state) && + poll_state_synchronize_rcu_full_unordered(&gp_state)) + return 1; + /* Has RCU gone idle with this CPU needing another grace period? */ if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp) && diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 7dfc57e9adb1..eedfa43059e8 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -253,7 +253,7 @@ struct rcu_data { u8 nocb_gp_sleep; /* Is the nocb GP thread asleep? */ u8 nocb_gp_bypass; /* Found a bypass on last scan? */ u8 nocb_gp_gp; /* GP to wait for on last scan? */ - unsigned long nocb_gp_seq; /* If so, ->gp_seq to wait for. */ + struct rcu_gp_seq nocb_gp_seq; /* If so, GP state to wait for. */ unsigned long nocb_gp_loops; /* # passes through wait code. */ struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */ bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */ @@ -296,6 +296,11 @@ struct rcu_data { int cpu; }; +static inline void rcu_defer_qs_clear(struct rcu_data *rdp) +{ + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); +} + /* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0 #define RCU_NOCB_WAKE_BYPASS 1 @@ -386,7 +391,6 @@ struct rcu_state { struct mutex exp_mutex; /* Serialize expedited GP. */ struct mutex exp_wake_mutex; /* Serialize wakeup. */ unsigned long expedited_sequence; /* Take a ticket. */ - atomic_t expedited_need_qs; /* # CPUs left to check in. */ struct swait_queue_head expedited_wq; /* Wait for check-ins. */ int ncpus_snap; /* # CPUs seen last time. */ u8 cbovld; /* Callback overload now? */ @@ -498,12 +502,13 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t); static void zero_cpu_stall_ticks(struct rcu_data *rdp); static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); +static void rcu_nocb_exp_cleanup(struct rcu_node *rnp); static void rcu_init_one_nocb(struct rcu_node *rnp); static bool wake_nocb_gp(struct rcu_data *rdp); static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j, bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy); + unsigned long flags, bool lazy); static void __maybe_unused __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, unsigned long flags); static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level); @@ -540,8 +545,7 @@ static bool rcu_nohz_full_cpu(void); static void record_gp_stall_check_time(void); static void rcu_iw_handler(struct irq_work *iwp); static void check_cpu_stall(struct rcu_data *rdp); -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay); +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay); /* Forward declarations for tree_exp.h. */ static void sync_rcu_do_polled_gp(struct work_struct *wp); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..682a3fb6087e 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -708,6 +708,8 @@ static void rcu_exp_wait_wake(unsigned long s) } smp_mb(); /* All above changes before wakeup. */ wake_up_all(&rnp->exp_wq[rcu_seq_ctr(s) & 0x3]); + if (rcu_is_leaf_node(rnp)) + rcu_nocb_exp_cleanup(rnp); } trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake")); mutex_unlock(&rcu_state.exp_wake_mutex); @@ -731,7 +733,7 @@ static void rcu_exp_need_qs(void) { lockdep_assert_irqs_disabled(); ASSERT_EXCLUSIVE_WRITER_SCOPED(*this_cpu_ptr(&rcu_data.cpu_no_qs.b.exp)); - __this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); + this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); /* Store .exp before .rcu_urgent_qs. */ smp_store_release(this_cpu_ptr(&rcu_data.rcu_urgent_qs), true); set_need_resched_current(); @@ -870,7 +872,7 @@ static void rcu_exp_handler(void *unused) ASSERT_EXCLUSIVE_WRITER_SCOPED(rdp->cpu_no_qs.b.exp); if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || - __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + this_cpu_read(rcu_data.cpu_no_qs.b.exp)) return; if (rcu_is_cpu_rrupt_from_idle() || (IS_ENABLED(CONFIG_PREEMPT_COUNT) && preempt_bh_enabled)) { @@ -1047,18 +1049,18 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited); /** * start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period - * @rgosp: Place to put snapshot of grace-period state + * @gsp: Place to put snapshot of grace-period state * - * Places the normal and expedited grace-period states in rgosp. This + * Places the normal and expedited grace-period states in gsp. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. * If the needed expedited grace period is not already slated to start, * initiates that grace period. */ -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - get_state_synchronize_rcu_full(rgosp); + get_state_synchronize_rcu_full(gsp); (void)start_poll_synchronize_rcu_expedited(); } EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited_full); @@ -1092,11 +1094,11 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); /** * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * @gsp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() * * If a full RCU grace period has elapsed since the call to * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), - * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * or start_poll_synchronize_rcu_expedited_full() from which @gsp was * obtained, just return. Otherwise, invoke synchronize_rcu_expedited() * to wait for a full grace period. * @@ -1107,12 +1109,12 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @rgosp and that returned at the end of + * to the function that provided @gsp and that returned at the end of * this function. */ -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - if (!poll_state_synchronize_rcu_full(rgosp)) + if (!poll_state_synchronize_rcu_full(gsp)) synchronize_rcu_expedited(); } EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 373b877cf171..19bb42672baf 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -170,13 +170,35 @@ static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) lockdep_assert_held(&rdp->nocb_lock); } +static void rcu_nocb_cleanup_wake(struct swait_queue_head *sq) +{ + if (swait_active(sq)) + swake_up_all(sq); +} + /* * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended * grace period. */ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) { - swake_up_all(sq); + /* + * swait_active() can be checked first because of the following + * ordering, which pairs the smp_mb() in rcu_gp_cleanup() against + * the implicit barrier in prepare_to_swait()/set_current_state() + * on the nocb_gp_wait() side: + * + * rcu_gp_cleanup() nocb_gp_wait() + * --------------- -------------- + * WRITE_ONCE(root->gp_seq, new_gp_seq); swait_event_interruptible_exclusive(sq) + * smp_mb() prepare_to_swait() + * if swait_active(sq) list_add_tail(...) + * swake_up_all(sq) set_current_state() + * smp_mb() + * if (poll_state_synchronize_rcu_full()) + * ... + */ + rcu_nocb_cleanup_wake(sq); } static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) @@ -190,6 +212,38 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) init_swait_queue_head(&rnp->nocb_gp_wq[1]); } +/* + * Wake NOCB rcuog kthreads on a leaf node so that they can advance + * callbacks that were waiting for the just-completed expedited GP. + * + * The rcuog kthread waiting for a grace period sleeps on the per-leaf-node + * ->nocb_gp_wq[] (not on its rdp_gp's ->nocb_gp_wq, which only signals that + * new callbacks have shown up), so this is the queue that must be woken. + * Both the even and odd waitqueues are woken because the expedited sequence + * does not share parity with the normal ->gp_seq the waiter indexed with. + */ +static void rcu_nocb_exp_cleanup(struct rcu_node *rnp) +{ + /* + * swait_active() can be checked first because of the following + * ordering, which pairs the smp_mb() in rcu_exp_wait_wake() against + * the implicit barrier in prepare_to_swait()/set_current_state() + * on the nocb_gp_wait() side: + * + * rcu_exp_wait_wake() nocb_gp_wait() + * --------------- -------------- + * rcu_seq_end(&rcu_state.expedited_sequence); swait_event_interruptible_exclusive(sq) + * smp_mb() prepare_to_swait() + * if swait_active(sq) list_add_tail(...) + * swake_up_all(sq) set_current_state() + * smp_mb() + * if (poll_state_synchronize_rcu_full()) + * ... + */ + rcu_nocb_cleanup_wake(&rnp->nocb_gp_wq[0]); + rcu_nocb_cleanup_wake(&rnp->nocb_gp_wq[1]); +} + /* Clear any pending deferred wakeup timer (nocb_gp_lock must be held). */ static void nocb_defer_wakeup_cancel(struct rcu_data *rdp_gp) { @@ -433,7 +487,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, bool lazy) { unsigned long c; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long j = jiffies; long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); long lazy_len = READ_ONCE(rdp->lazy_len); @@ -502,7 +556,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } if (j != rdp->nocb_gp_adv_time && rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { + poll_state_synchronize_rcu_full(&cur_gp_seq)) { rcu_advance_cbs_nowake(rdp->mynode, rdp); rdp->nocb_gp_adv_time = j; } @@ -603,13 +657,13 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { bool was_alldone; if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) { /* Not enqueued on bypass but locked, do regular enqueue */ - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ } } @@ -659,7 +713,6 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) { bool bypass = false; int __maybe_unused cpu = my_rdp->cpu; - unsigned long cur_gp_seq; unsigned long flags; bool gotcbs = false; unsigned long j = jiffies; @@ -669,7 +722,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) bool needwake_gp; struct rcu_data *rdp, *rdp_toggling = NULL; struct rcu_node *rnp; - unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning. + struct rcu_gp_seq wait_gp_seq = {0}; // Suppress "use uninitialized" warning. bool wasempty = false; /* @@ -693,6 +746,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) * won't be ignored for long. */ list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { + struct rcu_gp_seq cur_gp_seq; long bypass_ncbs; bool flush_bypass = false; long lazy_ncbs; @@ -731,21 +785,27 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL) || (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) { + poll_state_synchronize_rcu_full(&cur_gp_seq))) { raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ needwake_gp = rcu_advance_cbs(rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL); raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */ } - // Need to wait on some grace period? WARN_ON_ONCE(wasempty && !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)); + // Need to wait on some grace period? if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { - if (!needwait_gp || - ULONG_CMP_LT(cur_gp_seq, wait_gp_seq)) - wait_gp_seq = cur_gp_seq; + /* + * Track the earliest pending normal and expedited GP + * across the group so the wait below can be released by + * whichever completes first. + */ + if (!needwait_gp || ULONG_CMP_LT(cur_gp_seq.norm, wait_gp_seq.norm)) + wait_gp_seq.norm = cur_gp_seq.norm; + if (!needwait_gp || ULONG_CMP_LT(cur_gp_seq.exp, wait_gp_seq.exp)) + wait_gp_seq.exp = cur_gp_seq.exp; needwait_gp = true; trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("NeedWaitGP")); @@ -767,7 +827,8 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) my_rdp->nocb_gp_bypass = bypass; my_rdp->nocb_gp_gp = needwait_gp; - my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0; + if (needwait_gp) + my_rdp->nocb_gp_seq = wait_gp_seq; // At least one child with non-empty ->nocb_bypass, so set // timer in order to avoid stranding its callbacks. @@ -802,12 +863,12 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) nocb_gp_sleep(my_rdp, cpu); } else { rnp = my_rdp->mynode; - trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait")); + trace_rcu_this_gp(rnp, wait_gp_seq.norm, TPS("StartWait")); swait_event_interruptible_exclusive( - rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1], - rcu_seq_done(&rnp->gp_seq, wait_gp_seq) || + rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq.norm) & 0x1], + poll_state_synchronize_rcu_full(&wait_gp_seq) || !READ_ONCE(my_rdp->nocb_gp_sleep)); - trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait")); + trace_rcu_this_gp(rnp, wait_gp_seq.norm, TPS("EndWait")); } if (!rcu_nocb_poll) { @@ -841,7 +902,8 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) swake_up_one(&rdp_toggling->nocb_state_wq); } - my_rdp->nocb_gp_seq = -1; + my_rdp->nocb_gp_seq.norm = -1; + my_rdp->nocb_gp_seq.exp = -1; WARN_ON(signal_pending(current)); } @@ -877,7 +939,7 @@ static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) static void nocb_cb_wait(struct rcu_data *rdp) { struct rcu_segcblist *cblist = &rdp->cblist; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long flags; bool needwake_gp = false; struct rcu_node *rnp = rdp->mynode; @@ -919,7 +981,7 @@ static void nocb_cb_wait(struct rcu_data *rdp) lockdep_assert_irqs_enabled(); rcu_nocb_lock_irqsave(rdp, flags); if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq) && + poll_state_synchronize_rcu_full(&cur_gp_seq) && raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */ needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ @@ -1525,7 +1587,7 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) { struct rcu_node *rnp = rdp->mynode; - pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu %c CPU %d%s\n", + pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld/%ld rnp %d:%d %lu %c CPU %d%s\n", rdp->cpu, "kK"[!!rdp->nocb_gp_kthread], "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], @@ -1537,7 +1599,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) ".W"[swait_active(&rnp->nocb_gp_wq[1])], ".B"[!!rdp->nocb_gp_bypass], ".G"[!!rdp->nocb_gp_gp], - (long)rdp->nocb_gp_seq, + (long)rdp->nocb_gp_seq.norm, + (long)rdp->nocb_gp_seq.exp, rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, @@ -1548,8 +1611,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) static void show_rcu_nocb_state(struct rcu_data *rdp) { char bufd[22]; - char bufw[45]; - char bufr[45]; + char bufw[64]; + char bufr[64]; char bufn[22]; char bufb[22]; struct rcu_data *nocb_next_rdp; @@ -1569,9 +1632,12 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) nocb_entry_rdp); sprintf(bufd, "%ld", rsclp->seglen[RCU_DONE_TAIL]); - sprintf(bufw, "%ld(%ld)", rsclp->seglen[RCU_WAIT_TAIL], rsclp->gp_seq[RCU_WAIT_TAIL]); - sprintf(bufr, "%ld(%ld)", rsclp->seglen[RCU_NEXT_READY_TAIL], - rsclp->gp_seq[RCU_NEXT_READY_TAIL]); + sprintf(bufw, "%ld(%ld/%ld)", rsclp->seglen[RCU_WAIT_TAIL], + rsclp->gp_seq[RCU_WAIT_TAIL].norm, + rsclp->gp_seq[RCU_WAIT_TAIL].exp); + sprintf(bufr, "%ld(%ld/%ld)", rsclp->seglen[RCU_NEXT_READY_TAIL], + rsclp->gp_seq[RCU_NEXT_READY_TAIL].norm, + rsclp->gp_seq[RCU_NEXT_READY_TAIL].exp); sprintf(bufn, "%ld", rsclp->seglen[RCU_NEXT_TAIL]); sprintf(bufb, "%ld", rcu_cblist_n_cbs(&rdp->nocb_bypass)); pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%s%c%s%c%s%c%s%c%s q%ld %c CPU %d%s\n", @@ -1654,6 +1720,10 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) { } +static void rcu_nocb_exp_cleanup(struct rcu_node *rnp) +{ +} + static bool wake_nocb_gp(struct rcu_data *rdp) { return false; @@ -1666,7 +1736,7 @@ static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { WARN_ON_ONCE(1); /* Should be dead code! */ } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..743c16247fc0 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -298,11 +298,11 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n"); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { + if (this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { trace_rcu_grace_period(TPS("rcu_preempt"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */ WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false); } @@ -488,7 +488,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) rdp = this_cpu_ptr(&rcu_data); if (rdp->defer_qs_pending == DEFER_QS_PENDING) - rdp->defer_qs_pending = DEFER_QS_IDLE; + rcu_defer_qs_clear(rdp); /* * If RCU core is waiting for this CPU to exit its critical section, @@ -599,7 +599,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) */ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { - return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) || + return (this_cpu_read(rcu_data.cpu_no_qs.b.exp) || READ_ONCE(t->rcu_read_unlock_special.s)) && rcu_preempt_depth() == 0; } @@ -614,9 +614,35 @@ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) notrace void rcu_preempt_deferred_qs(struct task_struct *t) { unsigned long flags; + struct rcu_data *rdp; - if (!rcu_preempt_need_deferred_qs(t)) + if (!rcu_preempt_need_deferred_qs(t)) { + /* + * If we got here from a softirq/irq_work that fired while + * rcu_preempt_depth() > 0, the deferred-QS mechanism has been + * consumed without doing any work: rcu_preempt_need_deferred_qs() + * just returned false because the task is still in a reader, so + * the actual QS report has to wait for the next + * rcu_read_unlock(). + * + * Clear ->defer_qs_pending here so the next outer + * rcu_read_unlock_special() can re-arm a fresh mechanism (in + * particular the irq_work path, which the local_irq_enable() + * recovery boundary cannot itself reschedule from). + * + * Recursion safety: rcu_preempt_depth() > 0 means we are inside + * an outer reader, so any inner rcu_read_unlock() reached via + * tracing (bpf programs attached to trace points) brings + * nesting to outer (> 0), never to 0, so no recursive + * raise_softirq_irqoff()/irq_work_queue_on() can be triggered + * by this clear. + */ + if (rcu_preempt_depth() > 0) { + rdp = this_cpu_ptr(&rcu_data); + rcu_defer_qs_clear(rdp); + } return; + } local_irq_save(flags); rcu_preempt_deferred_qs_irqrestore(t, flags); } @@ -645,7 +671,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); + rcu_defer_qs_clear(rdp); } /* @@ -923,10 +949,10 @@ void rcu_read_unlock_strict(void) * * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to - * __rcu_read_unlock(). + * __rcu_read_unlock()). */ rdp = this_cpu_ptr(&rcu_data); - rdp->cpu_no_qs.b.norm = false; + WRITE_ONCE(rdp->cpu_no_qs.b.norm, false); rcu_report_qs_rdp(rdp); udelay(rcu_unlock_delay); } @@ -950,12 +976,12 @@ static void __init rcu_bootup_announce(void) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!"); - if (!__this_cpu_read(rcu_data.cpu_no_qs.s)) + if (!this_cpu_read(rcu_data.cpu_no_qs.s)) return; trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + if (this_cpu_read(rcu_data.cpu_no_qs.b.exp)) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); } @@ -970,7 +996,7 @@ void rcu_all_qs(void) { unsigned long flags; - if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + if (!READ_ONCE(*raw_cpu_ptr(&rcu_data.rcu_urgent_qs))) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ @@ -978,8 +1004,8 @@ void rcu_all_qs(void) preempt_enable(); return; } - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) { local_irq_save(flags); rcu_momentary_eqs(); local_irq_restore(flags); @@ -999,8 +1025,8 @@ void rcu_note_context_switch(bool preempt) /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) rcu_momentary_eqs(); out: rcu_tasks_qs(current, preempt); @@ -1320,6 +1346,41 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + bool ret; + struct rcu_node *rnp; + struct task_struct *t = current; + + preempt_disable(); // Stabilize ->rcu_blocked_node + rnp = t->rcu_blocked_node; + if (!rnp) + ret = false; + else + ret = (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t); + preempt_enable(); + return ret; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index cf7ae51cba40..20634edfb44d 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -573,13 +573,13 @@ static void rcu_check_gp_kthread_starvation(void) if (rcu_is_gp_kthread_starving(&j)) { cpu = gpk ? task_cpu(gpk) : -1; - pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n", + pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%c ->cpu=%d\n", rcu_state.name, j, (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - gpk ? data_race(READ_ONCE(gpk->__state)) : ~0, cpu); + gpk ? task_state_to_char(gpk) : '?', cpu); if (gpk) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); @@ -616,12 +616,12 @@ static void rcu_check_gp_kthread_expired_fqs_timer(void) time_after(jiffies, jiffies_fqs + RCU_STALL_MIGHT_MIN) && gpk && !READ_ONCE(gpk->on_rq)) { cpu = task_cpu(gpk); - pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n", + pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%c\n", rcu_state.name, (jiffies - jiffies_fqs), (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), // Diagnostic read gp_state_getname(RCU_GP_WAIT_FQS), RCU_GP_WAIT_FQS, - data_race(READ_ONCE(gpk->__state))); + task_state_to_char(gpk)); pr_err("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n", cpu, kstat_softirqs_cpu(TIMER_SOFTIRQ, cpu)); } @@ -927,20 +927,13 @@ bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) } EXPORT_SYMBOL_GPL(rcu_check_boost_fail); -/* - * Show the state of the grace-period kthreads. - */ -void show_rcu_gp_kthreads(void) +static noinline_for_stack void show_rcu_state(void) { - unsigned long cbs = 0; - int cpu; unsigned long j; unsigned long ja; unsigned long jr; unsigned long js; unsigned long jw; - struct rcu_data *rdp; - struct rcu_node *rnp; struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); j = jiffies; @@ -948,30 +941,48 @@ void show_rcu_gp_kthreads(void) jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity)); js = j - data_race(READ_ONCE(rcu_state.gp_start)); jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time)); - pr_info("%s: wait state: %s(%d) ->state: %#x ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", + pr_info("%s: wait state: %s(%d) ->state: %c ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", rcu_state.name, gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - t ? data_race(READ_ONCE(t->__state)) : 0x1ffff, t ? t->rt_priority : 0xffU, + t ? task_state_to_char(t) : '?', t ? t->rt_priority : 0xffU, js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)), (long)data_race(READ_ONCE(rcu_state.gp_seq)), (long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)), data_race(READ_ONCE(rcu_state.gp_max)), data_race(READ_ONCE(rcu_state.gp_flags))); +} + +static noinline_for_stack void show_rcu_node(struct rcu_node *rnp) +{ + pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", + rnp->grplo, rnp->grphi, + (long)data_race(READ_ONCE(rnp->gp_seq)), + (long)data_race(READ_ONCE(rnp->gp_seq_needed)), + data_race(READ_ONCE(rnp->qsmask)), + ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], + ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], + ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], + ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], + data_race(READ_ONCE(rnp->n_boosts))); +} + +/* + * Show the state of the grace-period kthreads. + */ +void show_rcu_gp_kthreads(void) +{ + unsigned long cbs = 0; + int cpu; + struct rcu_data *rdp; + struct rcu_node *rnp; + + show_rcu_state(); rcu_for_each_node_breadth_first(rnp) { if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) && !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) && !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks))) continue; - pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", - rnp->grplo, rnp->grphi, - (long)data_race(READ_ONCE(rnp->gp_seq)), - (long)data_race(READ_ONCE(rnp->gp_seq_needed)), - data_race(READ_ONCE(rnp->qsmask)), - ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], - ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], - ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], - ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], - data_race(READ_ONCE(rnp->n_boosts))); + show_rcu_node(rnp); if (!rcu_is_leaf_node(rnp)) continue; for_each_leaf_node_possible_cpu(rnp, cpu) { @@ -998,8 +1009,7 @@ EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); * This function checks for grace-period requests that fail to motivate * RCU to come out of its idle mode. */ -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay) +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay) { unsigned long flags; unsigned long j; @@ -1074,7 +1084,7 @@ void rcu_fwd_progress_check(unsigned long j) __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end))); preempt_disable(); rdp = this_cpu_ptr(&rcu_data); - rcu_check_gp_start_stall(rdp->mynode, rdp, j); + rcu_check_gp_start_stall(rdp->mynode, j); preempt_enable(); } for_each_possible_cpu(cpu) { diff --git a/kernel/torture.c b/kernel/torture.c index 77cb3589b19f..8c4e6b2fe8ba 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -577,6 +577,8 @@ static int torture_shuffle(void *arg) */ int torture_shuffle_init(long shuffint) { + int ret; + shuffle_interval = shuffint; shuffle_idle_cpu = -1; @@ -587,7 +589,10 @@ int torture_shuffle_init(long shuffint) } /* Create the shuffler thread */ - return torture_create_kthread(torture_shuffle, NULL, shuffler_task); + ret = torture_create_kthread(torture_shuffle, NULL, shuffler_task); + if (ret) + free_cpumask_var(shuffle_tmp_mask); + return ret; } EXPORT_SYMBOL_GPL(torture_shuffle_init); diff --git a/mm/slab_common.c b/mm/slab_common.c index 03ecac12cd86..657fd75776ea 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1339,7 +1339,7 @@ static struct workqueue_struct *rcu_reclaim_wq; */ struct kvfree_rcu_bulk_data { struct list_head list; - struct rcu_gp_oldstate gp_snap; + struct rcu_gp_seq gp_snap; unsigned long nr_records; void *records[] __counted_by(nr_records); }; @@ -1364,7 +1364,7 @@ struct kvfree_rcu_bulk_data { struct kfree_rcu_cpu_work { struct rcu_work rcu_work; struct rcu_head *head_free; - struct rcu_gp_oldstate head_free_gp_snap; + struct rcu_gp_seq head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; }; @@ -1572,7 +1572,7 @@ static void kfree_rcu_work(struct work_struct *work) struct rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; - struct rcu_gp_oldstate head_gp_snap; + struct rcu_gp_seq head_gp_snap; int i; krwp = container_of(to_rcu_work(work), diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..35954f38ff6a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -91,6 +91,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "srcu.c" #include "sync.c" #include "task.c" #include "time.c" diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000000..1a2f563640e0 --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..0d6a5f1300c3 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -21,6 +21,7 @@ pub mod poll; pub mod rcu; mod refcount; mod set_once; +pub mod srcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use completion::Completion; @@ -31,6 +32,7 @@ pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; +pub use srcu::Srcu; /// Represents a lockdep class. /// diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000000..723e5e277fd6 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +}