2 * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition
3 * Internal non-public definitions that provide either classic
4 * or preemptible semantics.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20 * Copyright (c) 2010 Linaro
22 * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
25 #include <linux/kthread.h>
26 #include <linux/debugfs.h>
27 #include <linux/seq_file.h>
29 #ifdef CONFIG_RCU_TRACE
30 #define RCU_TRACE(stmt) stmt
31 #else /* #ifdef CONFIG_RCU_TRACE */
32 #define RCU_TRACE(stmt)
33 #endif /* #else #ifdef CONFIG_RCU_TRACE */
35 /* Global control variables for rcupdate callback mechanism. */
37 struct rcu_head
*rcucblist
; /* List of pending callbacks (CBs). */
38 struct rcu_head
**donetail
; /* ->next pointer of last "done" CB. */
39 struct rcu_head
**curtail
; /* ->next pointer of last CB. */
40 RCU_TRACE(long qlen
); /* Number of pending CBs. */
43 /* Definition for rcupdate control block. */
44 static struct rcu_ctrlblk rcu_sched_ctrlblk
= {
45 .donetail
= &rcu_sched_ctrlblk
.rcucblist
,
46 .curtail
= &rcu_sched_ctrlblk
.rcucblist
,
49 static struct rcu_ctrlblk rcu_bh_ctrlblk
= {
50 .donetail
= &rcu_bh_ctrlblk
.rcucblist
,
51 .curtail
= &rcu_bh_ctrlblk
.rcucblist
,
54 #ifdef CONFIG_DEBUG_LOCK_ALLOC
55 int rcu_scheduler_active __read_mostly
;
56 EXPORT_SYMBOL_GPL(rcu_scheduler_active
);
57 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
59 #ifdef CONFIG_TINY_PREEMPT_RCU
61 #include <linux/delay.h>
63 /* Global control variables for preemptible RCU. */
64 struct rcu_preempt_ctrlblk
{
65 struct rcu_ctrlblk rcb
; /* curtail: ->next ptr of last CB for GP. */
66 struct rcu_head
**nexttail
;
67 /* Tasks blocked in a preemptible RCU */
68 /* read-side critical section while an */
69 /* preemptible-RCU grace period is in */
70 /* progress must wait for a later grace */
71 /* period. This pointer points to the */
72 /* ->next pointer of the last task that */
73 /* must wait for a later grace period, or */
74 /* to &->rcb.rcucblist if there is no */
76 struct list_head blkd_tasks
;
77 /* Tasks blocked in RCU read-side critical */
78 /* section. Tasks are placed at the head */
79 /* of this list and age towards the tail. */
80 struct list_head
*gp_tasks
;
81 /* Pointer to the first task blocking the */
82 /* current grace period, or NULL if there */
83 /* is no such task. */
84 struct list_head
*exp_tasks
;
85 /* Pointer to first task blocking the */
86 /* current expedited grace period, or NULL */
87 /* if there is no such task. If there */
88 /* is no current expedited grace period, */
89 /* then there cannot be any such task. */
90 #ifdef CONFIG_RCU_BOOST
91 struct list_head
*boost_tasks
;
92 /* Pointer to first task that needs to be */
93 /* priority-boosted, or NULL if no priority */
94 /* boosting is needed. If there is no */
95 /* current or expedited grace period, there */
96 /* can be no such task. */
97 #endif /* #ifdef CONFIG_RCU_BOOST */
98 u8 gpnum
; /* Current grace period. */
99 u8 gpcpu
; /* Last grace period blocked by the CPU. */
100 u8 completed
; /* Last grace period completed. */
101 /* If all three are equal, RCU is idle. */
102 #ifdef CONFIG_RCU_BOOST
103 unsigned long boost_time
; /* When to start boosting (jiffies) */
104 #endif /* #ifdef CONFIG_RCU_BOOST */
105 #ifdef CONFIG_RCU_TRACE
106 unsigned long n_grace_periods
;
107 #ifdef CONFIG_RCU_BOOST
108 unsigned long n_tasks_boosted
;
109 unsigned long n_exp_boosts
;
110 unsigned long n_normal_boosts
;
111 unsigned long n_normal_balk_blkd_tasks
;
112 unsigned long n_normal_balk_gp_tasks
;
113 unsigned long n_normal_balk_boost_tasks
;
114 unsigned long n_normal_balk_notyet
;
115 unsigned long n_normal_balk_nos
;
116 unsigned long n_exp_balk_blkd_tasks
;
117 unsigned long n_exp_balk_nos
;
118 #endif /* #ifdef CONFIG_RCU_BOOST */
119 #endif /* #ifdef CONFIG_RCU_TRACE */
122 static struct rcu_preempt_ctrlblk rcu_preempt_ctrlblk
= {
123 .rcb
.donetail
= &rcu_preempt_ctrlblk
.rcb
.rcucblist
,
124 .rcb
.curtail
= &rcu_preempt_ctrlblk
.rcb
.rcucblist
,
125 .nexttail
= &rcu_preempt_ctrlblk
.rcb
.rcucblist
,
126 .blkd_tasks
= LIST_HEAD_INIT(rcu_preempt_ctrlblk
.blkd_tasks
),
129 static int rcu_preempted_readers_exp(void);
130 static void rcu_report_exp_done(void);
133 * Return true if the CPU has not yet responded to the current grace period.
135 static int rcu_cpu_blocking_cur_gp(void)
137 return rcu_preempt_ctrlblk
.gpcpu
!= rcu_preempt_ctrlblk
.gpnum
;
141 * Check for a running RCU reader. Because there is only one CPU,
142 * there can be but one running RCU reader at a time. ;-)
144 static int rcu_preempt_running_reader(void)
146 return current
->rcu_read_lock_nesting
;
150 * Check for preempted RCU readers blocking any grace period.
151 * If the caller needs a reliable answer, it must disable hard irqs.
153 static int rcu_preempt_blocked_readers_any(void)
155 return !list_empty(&rcu_preempt_ctrlblk
.blkd_tasks
);
159 * Check for preempted RCU readers blocking the current grace period.
160 * If the caller needs a reliable answer, it must disable hard irqs.
162 static int rcu_preempt_blocked_readers_cgp(void)
164 return rcu_preempt_ctrlblk
.gp_tasks
!= NULL
;
168 * Return true if another preemptible-RCU grace period is needed.
170 static int rcu_preempt_needs_another_gp(void)
172 return *rcu_preempt_ctrlblk
.rcb
.curtail
!= NULL
;
176 * Return true if a preemptible-RCU grace period is in progress.
177 * The caller must disable hardirqs.
179 static int rcu_preempt_gp_in_progress(void)
181 return rcu_preempt_ctrlblk
.completed
!= rcu_preempt_ctrlblk
.gpnum
;
185 * Advance a ->blkd_tasks-list pointer to the next entry, instead
186 * returning NULL if at the end of the list.
188 static struct list_head
*rcu_next_node_entry(struct task_struct
*t
)
190 struct list_head
*np
;
192 np
= t
->rcu_node_entry
.next
;
193 if (np
== &rcu_preempt_ctrlblk
.blkd_tasks
)
198 #ifdef CONFIG_RCU_TRACE
200 #ifdef CONFIG_RCU_BOOST
201 static void rcu_initiate_boost_trace(void);
202 static void rcu_initiate_exp_boost_trace(void);
203 #endif /* #ifdef CONFIG_RCU_BOOST */
206 * Dump additional statistice for TINY_PREEMPT_RCU.
208 static void show_tiny_preempt_stats(struct seq_file
*m
)
210 seq_printf(m
, "rcu_preempt: qlen=%ld gp=%lu g%u/p%u/c%u tasks=%c%c%c\n",
211 rcu_preempt_ctrlblk
.rcb
.qlen
,
212 rcu_preempt_ctrlblk
.n_grace_periods
,
213 rcu_preempt_ctrlblk
.gpnum
,
214 rcu_preempt_ctrlblk
.gpcpu
,
215 rcu_preempt_ctrlblk
.completed
,
216 "T."[list_empty(&rcu_preempt_ctrlblk
.blkd_tasks
)],
217 "N."[!rcu_preempt_ctrlblk
.gp_tasks
],
218 "E."[!rcu_preempt_ctrlblk
.exp_tasks
]);
219 #ifdef CONFIG_RCU_BOOST
220 seq_printf(m
, "%sttb=%c ntb=%lu neb=%lu nnb=%lu j=%04x bt=%04x\n",
222 "B."[!rcu_preempt_ctrlblk
.boost_tasks
],
223 rcu_preempt_ctrlblk
.n_tasks_boosted
,
224 rcu_preempt_ctrlblk
.n_exp_boosts
,
225 rcu_preempt_ctrlblk
.n_normal_boosts
,
226 (int)(jiffies
& 0xffff),
227 (int)(rcu_preempt_ctrlblk
.boost_time
& 0xffff));
228 seq_printf(m
, " %s: nt=%lu gt=%lu bt=%lu ny=%lu nos=%lu\n",
230 rcu_preempt_ctrlblk
.n_normal_balk_blkd_tasks
,
231 rcu_preempt_ctrlblk
.n_normal_balk_gp_tasks
,
232 rcu_preempt_ctrlblk
.n_normal_balk_boost_tasks
,
233 rcu_preempt_ctrlblk
.n_normal_balk_notyet
,
234 rcu_preempt_ctrlblk
.n_normal_balk_nos
);
235 seq_printf(m
, " exp balk: bt=%lu nos=%lu\n",
236 rcu_preempt_ctrlblk
.n_exp_balk_blkd_tasks
,
237 rcu_preempt_ctrlblk
.n_exp_balk_nos
);
238 #endif /* #ifdef CONFIG_RCU_BOOST */
241 #endif /* #ifdef CONFIG_RCU_TRACE */
243 #ifdef CONFIG_RCU_BOOST
245 #include "rtmutex_common.h"
248 * Carry out RCU priority boosting on the task indicated by ->boost_tasks,
249 * and advance ->boost_tasks to the next task in the ->blkd_tasks list.
251 static int rcu_boost(void)
255 struct list_head
*np
;
256 struct task_struct
*t
;
258 if (rcu_preempt_ctrlblk
.boost_tasks
== NULL
)
259 return 0; /* Nothing to boost. */
260 raw_local_irq_save(flags
);
261 t
= container_of(rcu_preempt_ctrlblk
.boost_tasks
, struct task_struct
,
263 np
= rcu_next_node_entry(t
);
264 rt_mutex_init_proxy_locked(&mtx
, t
);
265 t
->rcu_boost_mutex
= &mtx
;
266 t
->rcu_read_unlock_special
|= RCU_READ_UNLOCK_BOOSTED
;
267 raw_local_irq_restore(flags
);
269 RCU_TRACE(rcu_preempt_ctrlblk
.n_tasks_boosted
++);
270 rt_mutex_unlock(&mtx
);
271 return rcu_preempt_ctrlblk
.boost_tasks
!= NULL
;
275 * Check to see if it is now time to start boosting RCU readers blocking
276 * the current grace period, and, if so, tell the rcu_kthread_task to
277 * start boosting them. If there is an expedited boost in progress,
278 * we wait for it to complete.
280 * If there are no blocked readers blocking the current grace period,
281 * return 0 to let the caller know, otherwise return 1. Note that this
282 * return value is independent of whether or not boosting was done.
284 static int rcu_initiate_boost(void)
286 if (!rcu_preempt_blocked_readers_cgp()) {
287 RCU_TRACE(rcu_preempt_ctrlblk
.n_normal_balk_blkd_tasks
++);
290 if (rcu_preempt_ctrlblk
.gp_tasks
!= NULL
&&
291 rcu_preempt_ctrlblk
.boost_tasks
== NULL
&&
292 ULONG_CMP_GE(jiffies
, rcu_preempt_ctrlblk
.boost_time
)) {
293 rcu_preempt_ctrlblk
.boost_tasks
= rcu_preempt_ctrlblk
.gp_tasks
;
294 invoke_rcu_kthread();
295 RCU_TRACE(rcu_preempt_ctrlblk
.n_normal_boosts
++);
297 RCU_TRACE(rcu_initiate_boost_trace());
302 * Initiate boosting for an expedited grace period.
304 static void rcu_initiate_expedited_boost(void)
308 raw_local_irq_save(flags
);
309 if (!list_empty(&rcu_preempt_ctrlblk
.blkd_tasks
)) {
310 rcu_preempt_ctrlblk
.boost_tasks
=
311 rcu_preempt_ctrlblk
.blkd_tasks
.next
;
312 invoke_rcu_kthread();
313 RCU_TRACE(rcu_preempt_ctrlblk
.n_exp_boosts
++);
315 RCU_TRACE(rcu_initiate_exp_boost_trace());
316 raw_local_irq_restore(flags
);
319 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
322 * Do priority-boost accounting for the start of a new grace period.
324 static void rcu_preempt_boost_start_gp(void)
326 rcu_preempt_ctrlblk
.boost_time
= jiffies
+ RCU_BOOST_DELAY_JIFFIES
;
329 #else /* #ifdef CONFIG_RCU_BOOST */
332 * If there is no RCU priority boosting, we don't boost.
334 static int rcu_boost(void)
340 * If there is no RCU priority boosting, we don't initiate boosting,
341 * but we do indicate whether there are blocked readers blocking the
342 * current grace period.
344 static int rcu_initiate_boost(void)
346 return rcu_preempt_blocked_readers_cgp();
350 * If there is no RCU priority boosting, we don't initiate expedited boosting.
352 static void rcu_initiate_expedited_boost(void)
357 * If there is no RCU priority boosting, nothing to do at grace-period start.
359 static void rcu_preempt_boost_start_gp(void)
363 #endif /* else #ifdef CONFIG_RCU_BOOST */
366 * Record a preemptible-RCU quiescent state for the specified CPU. Note
367 * that this just means that the task currently running on the CPU is
368 * in a quiescent state. There might be any number of tasks blocked
369 * while in an RCU read-side critical section.
371 * Unlike the other rcu_*_qs() functions, callers to this function
372 * must disable irqs in order to protect the assignment to
373 * ->rcu_read_unlock_special.
375 * Because this is a single-CPU implementation, the only way a grace
376 * period can end is if the CPU is in a quiescent state. The reason is
377 * that a blocked preemptible-RCU reader can exit its critical section
378 * only if the CPU is running it at the time. Therefore, when the
379 * last task blocking the current grace period exits its RCU read-side
380 * critical section, neither the CPU nor blocked tasks will be stopping
381 * the current grace period. (In contrast, SMP implementations
382 * might have CPUs running in RCU read-side critical sections that
383 * block later grace periods -- but this is not possible given only
386 static void rcu_preempt_cpu_qs(void)
388 /* Record both CPU and task as having responded to current GP. */
389 rcu_preempt_ctrlblk
.gpcpu
= rcu_preempt_ctrlblk
.gpnum
;
390 current
->rcu_read_unlock_special
&= ~RCU_READ_UNLOCK_NEED_QS
;
392 /* If there is no GP then there is nothing more to do. */
393 if (!rcu_preempt_gp_in_progress())
396 * Check up on boosting. If there are readers blocking the
397 * current grace period, leave.
399 if (rcu_initiate_boost())
402 /* Advance callbacks. */
403 rcu_preempt_ctrlblk
.completed
= rcu_preempt_ctrlblk
.gpnum
;
404 rcu_preempt_ctrlblk
.rcb
.donetail
= rcu_preempt_ctrlblk
.rcb
.curtail
;
405 rcu_preempt_ctrlblk
.rcb
.curtail
= rcu_preempt_ctrlblk
.nexttail
;
407 /* If there are no blocked readers, next GP is done instantly. */
408 if (!rcu_preempt_blocked_readers_any())
409 rcu_preempt_ctrlblk
.rcb
.donetail
= rcu_preempt_ctrlblk
.nexttail
;
411 /* If there are done callbacks, cause them to be invoked. */
412 if (*rcu_preempt_ctrlblk
.rcb
.donetail
!= NULL
)
413 invoke_rcu_kthread();
417 * Start a new RCU grace period if warranted. Hard irqs must be disabled.
419 static void rcu_preempt_start_gp(void)
421 if (!rcu_preempt_gp_in_progress() && rcu_preempt_needs_another_gp()) {
423 /* Official start of GP. */
424 rcu_preempt_ctrlblk
.gpnum
++;
425 RCU_TRACE(rcu_preempt_ctrlblk
.n_grace_periods
++);
427 /* Any blocked RCU readers block new GP. */
428 if (rcu_preempt_blocked_readers_any())
429 rcu_preempt_ctrlblk
.gp_tasks
=
430 rcu_preempt_ctrlblk
.blkd_tasks
.next
;
432 /* Set up for RCU priority boosting. */
433 rcu_preempt_boost_start_gp();
435 /* If there is no running reader, CPU is done with GP. */
436 if (!rcu_preempt_running_reader())
437 rcu_preempt_cpu_qs();
442 * We have entered the scheduler, and the current task might soon be
443 * context-switched away from. If this task is in an RCU read-side
444 * critical section, we will no longer be able to rely on the CPU to
445 * record that fact, so we enqueue the task on the blkd_tasks list.
446 * If the task started after the current grace period began, as recorded
447 * by ->gpcpu, we enqueue at the beginning of the list. Otherwise
448 * before the element referenced by ->gp_tasks (or at the tail if
449 * ->gp_tasks is NULL) and point ->gp_tasks at the newly added element.
450 * The task will dequeue itself when it exits the outermost enclosing
451 * RCU read-side critical section. Therefore, the current grace period
452 * cannot be permitted to complete until the ->gp_tasks pointer becomes
455 * Caller must disable preemption.
457 void rcu_preempt_note_context_switch(void)
459 struct task_struct
*t
= current
;
462 local_irq_save(flags
); /* must exclude scheduler_tick(). */
463 if (rcu_preempt_running_reader() &&
464 (t
->rcu_read_unlock_special
& RCU_READ_UNLOCK_BLOCKED
) == 0) {
466 /* Possibly blocking in an RCU read-side critical section. */
467 t
->rcu_read_unlock_special
|= RCU_READ_UNLOCK_BLOCKED
;
470 * If this CPU has already checked in, then this task
471 * will hold up the next grace period rather than the
472 * current grace period. Queue the task accordingly.
473 * If the task is queued for the current grace period
474 * (i.e., this CPU has not yet passed through a quiescent
475 * state for the current grace period), then as long
476 * as that task remains queued, the current grace period
479 list_add(&t
->rcu_node_entry
, &rcu_preempt_ctrlblk
.blkd_tasks
);
480 if (rcu_cpu_blocking_cur_gp())
481 rcu_preempt_ctrlblk
.gp_tasks
= &t
->rcu_node_entry
;
485 * Either we were not in an RCU read-side critical section to
486 * begin with, or we have now recorded that critical section
487 * globally. Either way, we can now note a quiescent state
488 * for this CPU. Again, if we were in an RCU read-side critical
489 * section, and if that critical section was blocking the current
490 * grace period, then the fact that the task has been enqueued
491 * means that current grace period continues to be blocked.
493 rcu_preempt_cpu_qs();
494 local_irq_restore(flags
);
498 * Tiny-preemptible RCU implementation for rcu_read_lock().
499 * Just increment ->rcu_read_lock_nesting, shared state will be updated
502 void __rcu_read_lock(void)
504 current
->rcu_read_lock_nesting
++;
505 barrier(); /* needed if we ever invoke rcu_read_lock in rcutiny.c */
507 EXPORT_SYMBOL_GPL(__rcu_read_lock
);
510 * Handle special cases during rcu_read_unlock(), such as needing to
511 * notify RCU core processing or task having blocked during the RCU
512 * read-side critical section.
514 static void rcu_read_unlock_special(struct task_struct
*t
)
519 struct list_head
*np
;
523 * NMI handlers cannot block and cannot safely manipulate state.
524 * They therefore cannot possibly be special, so just leave.
529 local_irq_save(flags
);
532 * If RCU core is waiting for this CPU to exit critical section,
533 * let it know that we have done so.
535 special
= t
->rcu_read_unlock_special
;
536 if (special
& RCU_READ_UNLOCK_NEED_QS
)
537 rcu_preempt_cpu_qs();
539 /* Hardware IRQ handlers cannot block. */
541 local_irq_restore(flags
);
545 /* Clean up if blocked during RCU read-side critical section. */
546 if (special
& RCU_READ_UNLOCK_BLOCKED
) {
547 t
->rcu_read_unlock_special
&= ~RCU_READ_UNLOCK_BLOCKED
;
550 * Remove this task from the ->blkd_tasks list and adjust
551 * any pointers that might have been referencing it.
553 empty
= !rcu_preempt_blocked_readers_cgp();
554 empty_exp
= rcu_preempt_ctrlblk
.exp_tasks
== NULL
;
555 np
= rcu_next_node_entry(t
);
556 list_del_init(&t
->rcu_node_entry
);
557 if (&t
->rcu_node_entry
== rcu_preempt_ctrlblk
.gp_tasks
)
558 rcu_preempt_ctrlblk
.gp_tasks
= np
;
559 if (&t
->rcu_node_entry
== rcu_preempt_ctrlblk
.exp_tasks
)
560 rcu_preempt_ctrlblk
.exp_tasks
= np
;
561 #ifdef CONFIG_RCU_BOOST
562 if (&t
->rcu_node_entry
== rcu_preempt_ctrlblk
.boost_tasks
)
563 rcu_preempt_ctrlblk
.boost_tasks
= np
;
564 #endif /* #ifdef CONFIG_RCU_BOOST */
567 * If this was the last task on the current list, and if
568 * we aren't waiting on the CPU, report the quiescent state
569 * and start a new grace period if needed.
571 if (!empty
&& !rcu_preempt_blocked_readers_cgp()) {
572 rcu_preempt_cpu_qs();
573 rcu_preempt_start_gp();
577 * If this was the last task on the expedited lists,
578 * then we need wake up the waiting task.
580 if (!empty_exp
&& rcu_preempt_ctrlblk
.exp_tasks
== NULL
)
581 rcu_report_exp_done();
583 #ifdef CONFIG_RCU_BOOST
584 /* Unboost self if was boosted. */
585 if (special
& RCU_READ_UNLOCK_BOOSTED
) {
586 t
->rcu_read_unlock_special
&= ~RCU_READ_UNLOCK_BOOSTED
;
587 rt_mutex_unlock(t
->rcu_boost_mutex
);
588 t
->rcu_boost_mutex
= NULL
;
590 #endif /* #ifdef CONFIG_RCU_BOOST */
591 local_irq_restore(flags
);
595 * Tiny-preemptible RCU implementation for rcu_read_unlock().
596 * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost
597 * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
598 * invoke rcu_read_unlock_special() to clean up after a context switch
599 * in an RCU read-side critical section and other special cases.
601 void __rcu_read_unlock(void)
603 struct task_struct
*t
= current
;
605 barrier(); /* needed if we ever invoke rcu_read_unlock in rcutiny.c */
606 --t
->rcu_read_lock_nesting
;
607 barrier(); /* decrement before load of ->rcu_read_unlock_special */
608 if (t
->rcu_read_lock_nesting
== 0 &&
609 unlikely(ACCESS_ONCE(t
->rcu_read_unlock_special
)))
610 rcu_read_unlock_special(t
);
611 #ifdef CONFIG_PROVE_LOCKING
612 WARN_ON_ONCE(t
->rcu_read_lock_nesting
< 0);
613 #endif /* #ifdef CONFIG_PROVE_LOCKING */
615 EXPORT_SYMBOL_GPL(__rcu_read_unlock
);
618 * Check for a quiescent state from the current CPU. When a task blocks,
619 * the task is recorded in the rcu_preempt_ctrlblk structure, which is
620 * checked elsewhere. This is called from the scheduling-clock interrupt.
622 * Caller must disable hard irqs.
624 static void rcu_preempt_check_callbacks(void)
626 struct task_struct
*t
= current
;
628 if (rcu_preempt_gp_in_progress() &&
629 (!rcu_preempt_running_reader() ||
630 !rcu_cpu_blocking_cur_gp()))
631 rcu_preempt_cpu_qs();
632 if (&rcu_preempt_ctrlblk
.rcb
.rcucblist
!=
633 rcu_preempt_ctrlblk
.rcb
.donetail
)
634 invoke_rcu_kthread();
635 if (rcu_preempt_gp_in_progress() &&
636 rcu_cpu_blocking_cur_gp() &&
637 rcu_preempt_running_reader())
638 t
->rcu_read_unlock_special
|= RCU_READ_UNLOCK_NEED_QS
;
642 * TINY_PREEMPT_RCU has an extra callback-list tail pointer to
643 * update, so this is invoked from rcu_process_callbacks() to
644 * handle that case. Of course, it is invoked for all flavors of
645 * RCU, but RCU callbacks can appear only on one of the lists, and
646 * neither ->nexttail nor ->donetail can possibly be NULL, so there
647 * is no need for an explicit check.
649 static void rcu_preempt_remove_callbacks(struct rcu_ctrlblk
*rcp
)
651 if (rcu_preempt_ctrlblk
.nexttail
== rcp
->donetail
)
652 rcu_preempt_ctrlblk
.nexttail
= &rcp
->rcucblist
;
656 * Process callbacks for preemptible RCU.
658 static void rcu_preempt_process_callbacks(void)
660 rcu_process_callbacks(&rcu_preempt_ctrlblk
.rcb
);
664 * Queue a preemptible -RCU callback for invocation after a grace period.
666 void call_rcu(struct rcu_head
*head
, void (*func
)(struct rcu_head
*rcu
))
670 debug_rcu_head_queue(head
);
674 local_irq_save(flags
);
675 *rcu_preempt_ctrlblk
.nexttail
= head
;
676 rcu_preempt_ctrlblk
.nexttail
= &head
->next
;
677 RCU_TRACE(rcu_preempt_ctrlblk
.rcb
.qlen
++);
678 rcu_preempt_start_gp(); /* checks to see if GP needed. */
679 local_irq_restore(flags
);
681 EXPORT_SYMBOL_GPL(call_rcu
);
683 void rcu_barrier(void)
685 struct rcu_synchronize rcu
;
687 init_rcu_head_on_stack(&rcu
.head
);
688 init_completion(&rcu
.completion
);
689 /* Will wake me after RCU finished. */
690 call_rcu(&rcu
.head
, wakeme_after_rcu
);
692 wait_for_completion(&rcu
.completion
);
693 destroy_rcu_head_on_stack(&rcu
.head
);
695 EXPORT_SYMBOL_GPL(rcu_barrier
);
698 * synchronize_rcu - wait until a grace period has elapsed.
700 * Control will return to the caller some time after a full grace
701 * period has elapsed, in other words after all currently executing RCU
702 * read-side critical sections have completed. RCU read-side critical
703 * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
706 void synchronize_rcu(void)
708 #ifdef CONFIG_DEBUG_LOCK_ALLOC
709 if (!rcu_scheduler_active
)
711 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
713 WARN_ON_ONCE(rcu_preempt_running_reader());
714 if (!rcu_preempt_blocked_readers_any())
717 /* Once we get past the fastpath checks, same code as rcu_barrier(). */
720 EXPORT_SYMBOL_GPL(synchronize_rcu
);
722 static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq
);
723 static unsigned long sync_rcu_preempt_exp_count
;
724 static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex
);
727 * Return non-zero if there are any tasks in RCU read-side critical
728 * sections blocking the current preemptible-RCU expedited grace period.
729 * If there is no preemptible-RCU expedited grace period currently in
730 * progress, returns zero unconditionally.
732 static int rcu_preempted_readers_exp(void)
734 return rcu_preempt_ctrlblk
.exp_tasks
!= NULL
;
738 * Report the exit from RCU read-side critical section for the last task
739 * that queued itself during or before the current expedited preemptible-RCU
742 static void rcu_report_exp_done(void)
744 wake_up(&sync_rcu_preempt_exp_wq
);
748 * Wait for an rcu-preempt grace period, but expedite it. The basic idea
749 * is to rely in the fact that there is but one CPU, and that it is
750 * illegal for a task to invoke synchronize_rcu_expedited() while in a
751 * preemptible-RCU read-side critical section. Therefore, any such
752 * critical sections must correspond to blocked tasks, which must therefore
753 * be on the ->blkd_tasks list. So just record the current head of the
754 * list in the ->exp_tasks pointer, and wait for all tasks including and
755 * after the task pointed to by ->exp_tasks to drain.
757 void synchronize_rcu_expedited(void)
760 struct rcu_preempt_ctrlblk
*rpcp
= &rcu_preempt_ctrlblk
;
763 barrier(); /* ensure prior action seen before grace period. */
765 WARN_ON_ONCE(rcu_preempt_running_reader());
768 * Acquire lock so that there is only one preemptible RCU grace
769 * period in flight. Of course, if someone does the expedited
770 * grace period for us while we are acquiring the lock, just leave.
772 snap
= sync_rcu_preempt_exp_count
+ 1;
773 mutex_lock(&sync_rcu_preempt_exp_mutex
);
774 if (ULONG_CMP_LT(snap
, sync_rcu_preempt_exp_count
))
775 goto unlock_mb_ret
; /* Others did our work for us. */
777 local_irq_save(flags
);
780 * All RCU readers have to already be on blkd_tasks because
781 * we cannot legally be executing in an RCU read-side critical
785 /* Snapshot current head of ->blkd_tasks list. */
786 rpcp
->exp_tasks
= rpcp
->blkd_tasks
.next
;
787 if (rpcp
->exp_tasks
== &rpcp
->blkd_tasks
)
788 rpcp
->exp_tasks
= NULL
;
789 local_irq_restore(flags
);
791 /* Wait for tail of ->blkd_tasks list to drain. */
792 if (rcu_preempted_readers_exp())
793 rcu_initiate_expedited_boost();
794 wait_event(sync_rcu_preempt_exp_wq
,
795 !rcu_preempted_readers_exp());
797 /* Clean up and exit. */
798 barrier(); /* ensure expedited GP seen before counter increment. */
799 sync_rcu_preempt_exp_count
++;
801 mutex_unlock(&sync_rcu_preempt_exp_mutex
);
802 barrier(); /* ensure subsequent action seen after grace period. */
804 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited
);
807 * Does preemptible RCU need the CPU to stay out of dynticks mode?
809 int rcu_preempt_needs_cpu(void)
811 if (!rcu_preempt_running_reader())
812 rcu_preempt_cpu_qs();
813 return rcu_preempt_ctrlblk
.rcb
.rcucblist
!= NULL
;
817 * Check for a task exiting while in a preemptible -RCU read-side
818 * critical section, clean up if so. No need to issue warnings,
819 * as debug_check_no_locks_held() already does this if lockdep
824 struct task_struct
*t
= current
;
826 if (t
->rcu_read_lock_nesting
== 0)
828 t
->rcu_read_lock_nesting
= 1;
832 #else /* #ifdef CONFIG_TINY_PREEMPT_RCU */
834 #ifdef CONFIG_RCU_TRACE
837 * Because preemptible RCU does not exist, it is not necessary to
838 * dump out its statistics.
840 static void show_tiny_preempt_stats(struct seq_file
*m
)
844 #endif /* #ifdef CONFIG_RCU_TRACE */
847 * Because preemptible RCU does not exist, it is never necessary to
848 * boost preempted RCU readers.
850 static int rcu_boost(void)
856 * Because preemptible RCU does not exist, it never has any callbacks
859 static void rcu_preempt_check_callbacks(void)
864 * Because preemptible RCU does not exist, it never has any callbacks
867 static void rcu_preempt_remove_callbacks(struct rcu_ctrlblk
*rcp
)
872 * Because preemptible RCU does not exist, it never has any callbacks
875 static void rcu_preempt_process_callbacks(void)
879 #endif /* #else #ifdef CONFIG_TINY_PREEMPT_RCU */
881 #ifdef CONFIG_DEBUG_LOCK_ALLOC
882 #include <linux/kernel_stat.h>
885 * During boot, we forgive RCU lockdep issues. After this function is
886 * invoked, we start taking RCU lockdep issues seriously.
888 void __init
rcu_scheduler_starting(void)
890 WARN_ON(nr_context_switches() > 0);
891 rcu_scheduler_active
= 1;
894 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
896 #ifdef CONFIG_RCU_BOOST
897 #define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO
898 #else /* #ifdef CONFIG_RCU_BOOST */
899 #define RCU_BOOST_PRIO 1
900 #endif /* #else #ifdef CONFIG_RCU_BOOST */
902 #ifdef CONFIG_RCU_TRACE
904 #ifdef CONFIG_RCU_BOOST
906 static void rcu_initiate_boost_trace(void)
908 if (rcu_preempt_ctrlblk
.gp_tasks
== NULL
)
909 rcu_preempt_ctrlblk
.n_normal_balk_gp_tasks
++;
910 else if (rcu_preempt_ctrlblk
.boost_tasks
!= NULL
)
911 rcu_preempt_ctrlblk
.n_normal_balk_boost_tasks
++;
912 else if (!ULONG_CMP_GE(jiffies
, rcu_preempt_ctrlblk
.boost_time
))
913 rcu_preempt_ctrlblk
.n_normal_balk_notyet
++;
915 rcu_preempt_ctrlblk
.n_normal_balk_nos
++;
918 static void rcu_initiate_exp_boost_trace(void)
920 if (list_empty(&rcu_preempt_ctrlblk
.blkd_tasks
))
921 rcu_preempt_ctrlblk
.n_exp_balk_blkd_tasks
++;
923 rcu_preempt_ctrlblk
.n_exp_balk_nos
++;
926 #endif /* #ifdef CONFIG_RCU_BOOST */
928 static void rcu_trace_sub_qlen(struct rcu_ctrlblk
*rcp
, int n
)
932 raw_local_irq_save(flags
);
934 raw_local_irq_restore(flags
);
938 * Dump statistics for TINY_RCU, such as they are.
940 static int show_tiny_stats(struct seq_file
*m
, void *unused
)
942 show_tiny_preempt_stats(m
);
943 seq_printf(m
, "rcu_sched: qlen: %ld\n", rcu_sched_ctrlblk
.qlen
);
944 seq_printf(m
, "rcu_bh: qlen: %ld\n", rcu_bh_ctrlblk
.qlen
);
948 static int show_tiny_stats_open(struct inode
*inode
, struct file
*file
)
950 return single_open(file
, show_tiny_stats
, NULL
);
953 static const struct file_operations show_tiny_stats_fops
= {
954 .owner
= THIS_MODULE
,
955 .open
= show_tiny_stats_open
,
958 .release
= single_release
,
961 static struct dentry
*rcudir
;
963 static int __init
rcutiny_trace_init(void)
965 struct dentry
*retval
;
967 rcudir
= debugfs_create_dir("rcu", NULL
);
970 retval
= debugfs_create_file("rcudata", 0444, rcudir
,
971 NULL
, &show_tiny_stats_fops
);
976 debugfs_remove_recursive(rcudir
);
980 static void __exit
rcutiny_trace_cleanup(void)
982 debugfs_remove_recursive(rcudir
);
985 module_init(rcutiny_trace_init
);
986 module_exit(rcutiny_trace_cleanup
);
988 MODULE_AUTHOR("Paul E. McKenney");
989 MODULE_DESCRIPTION("Read-Copy Update tracing for tiny implementation");
990 MODULE_LICENSE("GPL");
992 #endif /* #ifdef CONFIG_RCU_TRACE */