Commit | Line | Data |
---|---|---|
320ae51f JA |
1 | #include <linux/kernel.h> |
2 | #include <linux/module.h> | |
3 | #include <linux/init.h> | |
4 | #include <linux/blkdev.h> | |
5 | #include <linux/list.h> | |
6 | #include <linux/llist.h> | |
7 | #include <linux/smp.h> | |
8 | #include <linux/cpu.h> | |
9 | ||
10 | #include <linux/blk-mq.h> | |
11 | #include "blk-mq.h" | |
12 | ||
13 | static LIST_HEAD(blk_mq_cpu_notify_list); | |
14 | static DEFINE_SPINLOCK(blk_mq_cpu_notify_lock); | |
15 | ||
f618ef7c PG |
16 | static int blk_mq_main_cpu_notify(struct notifier_block *self, |
17 | unsigned long action, void *hcpu) | |
320ae51f JA |
18 | { |
19 | unsigned int cpu = (unsigned long) hcpu; | |
20 | struct blk_mq_cpu_notifier *notify; | |
21 | ||
22 | spin_lock(&blk_mq_cpu_notify_lock); | |
23 | ||
24 | list_for_each_entry(notify, &blk_mq_cpu_notify_list, list) | |
25 | notify->notify(notify->data, action, cpu); | |
26 | ||
27 | spin_unlock(&blk_mq_cpu_notify_lock); | |
28 | return NOTIFY_OK; | |
29 | } | |
30 | ||
f618ef7c PG |
31 | static void blk_mq_cpu_notify(void *data, unsigned long action, |
32 | unsigned int cpu) | |
320ae51f JA |
33 | { |
34 | if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { | |
35 | /* | |
36 | * If the CPU goes away, ensure that we run any pending | |
37 | * completions. | |
38 | */ | |
39 | struct llist_node *node; | |
40 | struct request *rq; | |
41 | ||
42 | local_irq_disable(); | |
43 | ||
44 | node = llist_del_all(&per_cpu(ipi_lists, cpu)); | |
45 | while (node) { | |
46 | struct llist_node *next = node->next; | |
47 | ||
48 | rq = llist_entry(node, struct request, ll_list); | |
49 | __blk_mq_end_io(rq, rq->errors); | |
50 | node = next; | |
51 | } | |
52 | ||
53 | local_irq_enable(); | |
54 | } | |
55 | } | |
56 | ||
57 | static struct notifier_block __cpuinitdata blk_mq_main_cpu_notifier = { | |
58 | .notifier_call = blk_mq_main_cpu_notify, | |
59 | }; | |
60 | ||
61 | void blk_mq_register_cpu_notifier(struct blk_mq_cpu_notifier *notifier) | |
62 | { | |
63 | BUG_ON(!notifier->notify); | |
64 | ||
65 | spin_lock(&blk_mq_cpu_notify_lock); | |
66 | list_add_tail(¬ifier->list, &blk_mq_cpu_notify_list); | |
67 | spin_unlock(&blk_mq_cpu_notify_lock); | |
68 | } | |
69 | ||
70 | void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier) | |
71 | { | |
72 | spin_lock(&blk_mq_cpu_notify_lock); | |
73 | list_del(¬ifier->list); | |
74 | spin_unlock(&blk_mq_cpu_notify_lock); | |
75 | } | |
76 | ||
77 | void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier, | |
78 | void (*fn)(void *, unsigned long, unsigned int), | |
79 | void *data) | |
80 | { | |
81 | notifier->notify = fn; | |
82 | notifier->data = data; | |
83 | } | |
84 | ||
85 | static struct blk_mq_cpu_notifier __cpuinitdata cpu_notifier = { | |
86 | .notify = blk_mq_cpu_notify, | |
87 | }; | |
88 | ||
89 | void __init blk_mq_cpu_init(void) | |
90 | { | |
91 | register_hotcpu_notifier(&blk_mq_main_cpu_notifier); | |
92 | blk_mq_register_cpu_notifier(&cpu_notifier); | |
93 | } |