Commit | Line | Data |
---|---|---|
ffdb5976 | 1 | /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
e5582ca2 RR |
2 | * GPL v2 and any later version. |
3 | */ | |
1da177e4 LT |
4 | #include <linux/cpu.h> |
5 | #include <linux/err.h> | |
ee527cd3 PB |
6 | #include <linux/kthread.h> |
7 | #include <linux/module.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/stop_machine.h> | |
1da177e4 | 10 | #include <linux/syscalls.h> |
a12bb444 BH |
11 | #include <linux/interrupt.h> |
12 | ||
1da177e4 | 13 | #include <asm/atomic.h> |
1da177e4 LT |
14 | #include <asm/uaccess.h> |
15 | ||
ffdb5976 | 16 | /* This controls the threads on each CPU. */ |
1da177e4 | 17 | enum stopmachine_state { |
ffdb5976 RR |
18 | /* Dummy starting state for thread. */ |
19 | STOPMACHINE_NONE, | |
20 | /* Awaiting everyone to be scheduled. */ | |
1da177e4 | 21 | STOPMACHINE_PREPARE, |
ffdb5976 | 22 | /* Disable interrupts. */ |
1da177e4 | 23 | STOPMACHINE_DISABLE_IRQ, |
ffdb5976 | 24 | /* Run the function */ |
5c2aed62 | 25 | STOPMACHINE_RUN, |
ffdb5976 | 26 | /* Exit */ |
1da177e4 LT |
27 | STOPMACHINE_EXIT, |
28 | }; | |
ffdb5976 | 29 | static enum stopmachine_state state; |
1da177e4 | 30 | |
5c2aed62 JB |
31 | struct stop_machine_data { |
32 | int (*fn)(void *); | |
33 | void *data; | |
ffdb5976 RR |
34 | int fnret; |
35 | }; | |
5c2aed62 | 36 | |
ffdb5976 RR |
37 | /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ |
38 | static unsigned int num_threads; | |
39 | static atomic_t thread_ack; | |
40 | static struct completion finished; | |
41 | static DEFINE_MUTEX(lock); | |
1da177e4 | 42 | |
ffdb5976 | 43 | static void set_state(enum stopmachine_state newstate) |
1da177e4 | 44 | { |
ffdb5976 RR |
45 | /* Reset ack counter. */ |
46 | atomic_set(&thread_ack, num_threads); | |
47 | smp_wmb(); | |
48 | state = newstate; | |
1da177e4 LT |
49 | } |
50 | ||
ffdb5976 RR |
51 | /* Last one to ack a state moves to the next state. */ |
52 | static void ack_state(void) | |
1da177e4 | 53 | { |
ffdb5976 RR |
54 | if (atomic_dec_and_test(&thread_ack)) { |
55 | /* If we're the last one to ack the EXIT, we're finished. */ | |
56 | if (state == STOPMACHINE_EXIT) | |
57 | complete(&finished); | |
58 | else | |
59 | set_state(state + 1); | |
60 | } | |
1da177e4 LT |
61 | } |
62 | ||
ffdb5976 RR |
63 | /* This is the actual thread which stops the CPU. It exits by itself rather |
64 | * than waiting for kthread_stop(), because it's easier for hotplug CPU. */ | |
65 | static int stop_cpu(struct stop_machine_data *smdata) | |
1da177e4 | 66 | { |
ffdb5976 | 67 | enum stopmachine_state curstate = STOPMACHINE_NONE; |
1da177e4 | 68 | |
ffdb5976 RR |
69 | /* Simple state machine */ |
70 | do { | |
71 | /* Chill out and ensure we re-read stopmachine_state. */ | |
3401a61e | 72 | cpu_relax(); |
ffdb5976 RR |
73 | if (state != curstate) { |
74 | curstate = state; | |
75 | switch (curstate) { | |
76 | case STOPMACHINE_DISABLE_IRQ: | |
77 | local_irq_disable(); | |
78 | hard_irq_disable(); | |
79 | break; | |
80 | case STOPMACHINE_RUN: | |
81 | /* |= allows error detection if functions on | |
82 | * multiple CPUs. */ | |
83 | smdata->fnret |= smdata->fn(smdata->data); | |
84 | break; | |
85 | default: | |
86 | break; | |
87 | } | |
88 | ack_state(); | |
89 | } | |
90 | } while (curstate != STOPMACHINE_EXIT); | |
1da177e4 | 91 | |
1da177e4 | 92 | local_irq_enable(); |
ffdb5976 | 93 | do_exit(0); |
1da177e4 LT |
94 | } |
95 | ||
ffdb5976 RR |
96 | /* Callback for CPUs which aren't supposed to do anything. */ |
97 | static int chill(void *unused) | |
5c2aed62 | 98 | { |
ffdb5976 | 99 | return 0; |
5c2aed62 | 100 | } |
1da177e4 | 101 | |
eeec4fad | 102 | int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
1da177e4 | 103 | { |
ffdb5976 RR |
104 | int i, err; |
105 | struct stop_machine_data active, idle; | |
106 | struct task_struct **threads; | |
107 | ||
108 | active.fn = fn; | |
109 | active.data = data; | |
110 | active.fnret = 0; | |
111 | idle.fn = chill; | |
112 | idle.data = NULL; | |
113 | ||
ffdb5976 RR |
114 | /* This could be too big for stack on large machines. */ |
115 | threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL); | |
116 | if (!threads) | |
117 | return -ENOMEM; | |
118 | ||
119 | /* Set up initial state. */ | |
120 | mutex_lock(&lock); | |
121 | init_completion(&finished); | |
122 | num_threads = num_online_cpus(); | |
123 | set_state(STOPMACHINE_PREPARE); | |
1da177e4 | 124 | |
ffdb5976 | 125 | for_each_online_cpu(i) { |
eeec4fad | 126 | struct stop_machine_data *smdata = &idle; |
ffdb5976 | 127 | struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; |
1da177e4 | 128 | |
eeec4fad RR |
129 | if (!cpus) { |
130 | if (i == first_cpu(cpu_online_map)) | |
131 | smdata = &active; | |
132 | } else { | |
133 | if (cpu_isset(i, *cpus)) | |
134 | smdata = &active; | |
135 | } | |
ffdb5976 RR |
136 | |
137 | threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u", | |
138 | i); | |
139 | if (IS_ERR(threads[i])) { | |
140 | err = PTR_ERR(threads[i]); | |
141 | threads[i] = NULL; | |
142 | goto kill_threads; | |
143 | } | |
1da177e4 | 144 | |
ffdb5976 RR |
145 | /* Place it onto correct cpu. */ |
146 | kthread_bind(threads[i], i); | |
1da177e4 | 147 | |
ffdb5976 RR |
148 | /* Make it highest prio. */ |
149 | if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, ¶m)) | |
150 | BUG(); | |
151 | } | |
1da177e4 | 152 | |
ffdb5976 RR |
153 | /* We've created all the threads. Wake them all: hold this CPU so one |
154 | * doesn't hit this CPU until we're ready. */ | |
eeec4fad | 155 | get_cpu(); |
ffdb5976 RR |
156 | for_each_online_cpu(i) |
157 | wake_up_process(threads[i]); | |
5c2aed62 | 158 | |
ffdb5976 RR |
159 | /* This will release the thread on our CPU. */ |
160 | put_cpu(); | |
161 | wait_for_completion(&finished); | |
162 | mutex_unlock(&lock); | |
1da177e4 | 163 | |
ffdb5976 | 164 | kfree(threads); |
1da177e4 | 165 | |
ffdb5976 | 166 | return active.fnret; |
1da177e4 | 167 | |
ffdb5976 RR |
168 | kill_threads: |
169 | for_each_online_cpu(i) | |
170 | if (threads[i]) | |
171 | kthread_stop(threads[i]); | |
172 | mutex_unlock(&lock); | |
85653af7 | 173 | |
ffdb5976 RR |
174 | kfree(threads); |
175 | return err; | |
1da177e4 LT |
176 | } |
177 | ||
eeec4fad | 178 | int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
1da177e4 | 179 | { |
1da177e4 LT |
180 | int ret; |
181 | ||
182 | /* No CPUs can come up or down during this. */ | |
86ef5c9a | 183 | get_online_cpus(); |
eeec4fad | 184 | ret = __stop_machine(fn, data, cpus); |
86ef5c9a | 185 | put_online_cpus(); |
1da177e4 LT |
186 | |
187 | return ret; | |
188 | } | |
eeec4fad | 189 | EXPORT_SYMBOL_GPL(stop_machine); |