Commit | Line | Data |
---|---|---|
38498a67 TG |
1 | /* |
2 | * Common SMP CPU bringup/teardown functions | |
3 | */ | |
f97f8f06 | 4 | #include <linux/cpu.h> |
29d5e047 TG |
5 | #include <linux/err.h> |
6 | #include <linux/smp.h> | |
8038dad7 | 7 | #include <linux/delay.h> |
38498a67 | 8 | #include <linux/init.h> |
f97f8f06 TG |
9 | #include <linux/list.h> |
10 | #include <linux/slab.h> | |
29d5e047 | 11 | #include <linux/sched.h> |
f97f8f06 | 12 | #include <linux/export.h> |
29d5e047 | 13 | #include <linux/percpu.h> |
f97f8f06 TG |
14 | #include <linux/kthread.h> |
15 | #include <linux/smpboot.h> | |
38498a67 TG |
16 | |
17 | #include "smpboot.h" | |
18 | ||
3180d89b PM |
19 | #ifdef CONFIG_SMP |
20 | ||
29d5e047 | 21 | #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD |
29d5e047 TG |
22 | /* |
23 | * For the hotplug case we keep the task structs around and reuse | |
24 | * them. | |
25 | */ | |
26 | static DEFINE_PER_CPU(struct task_struct *, idle_threads); | |
27 | ||
0db0628d | 28 | struct task_struct *idle_thread_get(unsigned int cpu) |
29d5e047 TG |
29 | { |
30 | struct task_struct *tsk = per_cpu(idle_threads, cpu); | |
31 | ||
32 | if (!tsk) | |
3bb5d2ee | 33 | return ERR_PTR(-ENOMEM); |
29d5e047 TG |
34 | init_idle(tsk, cpu); |
35 | return tsk; | |
36 | } | |
37 | ||
3bb5d2ee | 38 | void __init idle_thread_set_boot_cpu(void) |
29d5e047 | 39 | { |
3bb5d2ee | 40 | per_cpu(idle_threads, smp_processor_id()) = current; |
29d5e047 TG |
41 | } |
42 | ||
4a70d2d9 SB |
43 | /** |
44 | * idle_init - Initialize the idle thread for a cpu | |
45 | * @cpu: The cpu for which the idle thread should be initialized | |
46 | * | |
47 | * Creates the thread if it does not exist. | |
48 | */ | |
3bb5d2ee | 49 | static inline void idle_init(unsigned int cpu) |
29d5e047 | 50 | { |
3bb5d2ee SS |
51 | struct task_struct *tsk = per_cpu(idle_threads, cpu); |
52 | ||
53 | if (!tsk) { | |
54 | tsk = fork_idle(cpu); | |
55 | if (IS_ERR(tsk)) | |
56 | pr_err("SMP: fork_idle() failed for CPU %u\n", cpu); | |
57 | else | |
58 | per_cpu(idle_threads, cpu) = tsk; | |
59 | } | |
29d5e047 TG |
60 | } |
61 | ||
62 | /** | |
4a70d2d9 | 63 | * idle_threads_init - Initialize idle threads for all cpus |
29d5e047 | 64 | */ |
3bb5d2ee | 65 | void __init idle_threads_init(void) |
29d5e047 | 66 | { |
ee74d132 SB |
67 | unsigned int cpu, boot_cpu; |
68 | ||
69 | boot_cpu = smp_processor_id(); | |
29d5e047 | 70 | |
3bb5d2ee | 71 | for_each_possible_cpu(cpu) { |
ee74d132 | 72 | if (cpu != boot_cpu) |
3bb5d2ee | 73 | idle_init(cpu); |
29d5e047 | 74 | } |
29d5e047 | 75 | } |
29d5e047 | 76 | #endif |
f97f8f06 | 77 | |
3180d89b PM |
78 | #endif /* #ifdef CONFIG_SMP */ |
79 | ||
f97f8f06 TG |
80 | static LIST_HEAD(hotplug_threads); |
81 | static DEFINE_MUTEX(smpboot_threads_lock); | |
82 | ||
83 | struct smpboot_thread_data { | |
84 | unsigned int cpu; | |
85 | unsigned int status; | |
86 | struct smp_hotplug_thread *ht; | |
87 | }; | |
88 | ||
89 | enum { | |
90 | HP_THREAD_NONE = 0, | |
91 | HP_THREAD_ACTIVE, | |
92 | HP_THREAD_PARKED, | |
93 | }; | |
94 | ||
95 | /** | |
96 | * smpboot_thread_fn - percpu hotplug thread loop function | |
97 | * @data: thread data pointer | |
98 | * | |
99 | * Checks for thread stop and park conditions. Calls the necessary | |
100 | * setup, cleanup, park and unpark functions for the registered | |
101 | * thread. | |
102 | * | |
103 | * Returns 1 when the thread should exit, 0 otherwise. | |
104 | */ | |
105 | static int smpboot_thread_fn(void *data) | |
106 | { | |
107 | struct smpboot_thread_data *td = data; | |
108 | struct smp_hotplug_thread *ht = td->ht; | |
109 | ||
110 | while (1) { | |
111 | set_current_state(TASK_INTERRUPTIBLE); | |
112 | preempt_disable(); | |
113 | if (kthread_should_stop()) { | |
7d4d2696 | 114 | __set_current_state(TASK_RUNNING); |
f97f8f06 | 115 | preempt_enable(); |
3dd08c0c FW |
116 | /* cleanup must mirror setup */ |
117 | if (ht->cleanup && td->status != HP_THREAD_NONE) | |
f97f8f06 TG |
118 | ht->cleanup(td->cpu, cpu_online(td->cpu)); |
119 | kfree(td); | |
120 | return 0; | |
121 | } | |
122 | ||
123 | if (kthread_should_park()) { | |
124 | __set_current_state(TASK_RUNNING); | |
125 | preempt_enable(); | |
126 | if (ht->park && td->status == HP_THREAD_ACTIVE) { | |
127 | BUG_ON(td->cpu != smp_processor_id()); | |
128 | ht->park(td->cpu); | |
129 | td->status = HP_THREAD_PARKED; | |
130 | } | |
131 | kthread_parkme(); | |
132 | /* We might have been woken for stop */ | |
133 | continue; | |
134 | } | |
135 | ||
dc893e19 | 136 | BUG_ON(td->cpu != smp_processor_id()); |
f97f8f06 TG |
137 | |
138 | /* Check for state change setup */ | |
139 | switch (td->status) { | |
140 | case HP_THREAD_NONE: | |
7d4d2696 | 141 | __set_current_state(TASK_RUNNING); |
f97f8f06 TG |
142 | preempt_enable(); |
143 | if (ht->setup) | |
144 | ht->setup(td->cpu); | |
145 | td->status = HP_THREAD_ACTIVE; | |
7d4d2696 PZ |
146 | continue; |
147 | ||
f97f8f06 | 148 | case HP_THREAD_PARKED: |
7d4d2696 | 149 | __set_current_state(TASK_RUNNING); |
f97f8f06 TG |
150 | preempt_enable(); |
151 | if (ht->unpark) | |
152 | ht->unpark(td->cpu); | |
153 | td->status = HP_THREAD_ACTIVE; | |
7d4d2696 | 154 | continue; |
f97f8f06 TG |
155 | } |
156 | ||
157 | if (!ht->thread_should_run(td->cpu)) { | |
7d4d2696 | 158 | preempt_enable_no_resched(); |
f97f8f06 TG |
159 | schedule(); |
160 | } else { | |
7d4d2696 | 161 | __set_current_state(TASK_RUNNING); |
f97f8f06 TG |
162 | preempt_enable(); |
163 | ht->thread_fn(td->cpu); | |
164 | } | |
165 | } | |
166 | } | |
167 | ||
168 | static int | |
169 | __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu) | |
170 | { | |
171 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
172 | struct smpboot_thread_data *td; | |
173 | ||
174 | if (tsk) | |
175 | return 0; | |
176 | ||
177 | td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu)); | |
178 | if (!td) | |
179 | return -ENOMEM; | |
180 | td->cpu = cpu; | |
181 | td->ht = ht; | |
182 | ||
183 | tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu, | |
184 | ht->thread_comm); | |
185 | if (IS_ERR(tsk)) { | |
186 | kfree(td); | |
187 | return PTR_ERR(tsk); | |
188 | } | |
f97f8f06 TG |
189 | get_task_struct(tsk); |
190 | *per_cpu_ptr(ht->store, cpu) = tsk; | |
f2530dc7 TG |
191 | if (ht->create) { |
192 | /* | |
193 | * Make sure that the task has actually scheduled out | |
194 | * into park position, before calling the create | |
195 | * callback. At least the migration thread callback | |
196 | * requires that the task is off the runqueue. | |
197 | */ | |
198 | if (!wait_task_inactive(tsk, TASK_PARKED)) | |
199 | WARN_ON(1); | |
200 | else | |
201 | ht->create(cpu); | |
202 | } | |
f97f8f06 TG |
203 | return 0; |
204 | } | |
205 | ||
206 | int smpboot_create_threads(unsigned int cpu) | |
207 | { | |
208 | struct smp_hotplug_thread *cur; | |
209 | int ret = 0; | |
210 | ||
211 | mutex_lock(&smpboot_threads_lock); | |
212 | list_for_each_entry(cur, &hotplug_threads, list) { | |
213 | ret = __smpboot_create_thread(cur, cpu); | |
214 | if (ret) | |
215 | break; | |
216 | } | |
217 | mutex_unlock(&smpboot_threads_lock); | |
218 | return ret; | |
219 | } | |
220 | ||
221 | static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu) | |
222 | { | |
223 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
224 | ||
46c498c2 TG |
225 | if (ht->pre_unpark) |
226 | ht->pre_unpark(cpu); | |
f97f8f06 TG |
227 | kthread_unpark(tsk); |
228 | } | |
229 | ||
230 | void smpboot_unpark_threads(unsigned int cpu) | |
231 | { | |
232 | struct smp_hotplug_thread *cur; | |
233 | ||
234 | mutex_lock(&smpboot_threads_lock); | |
235 | list_for_each_entry(cur, &hotplug_threads, list) | |
b5242e98 CM |
236 | if (cpumask_test_cpu(cpu, cur->cpumask)) |
237 | smpboot_unpark_thread(cur, cpu); | |
f97f8f06 TG |
238 | mutex_unlock(&smpboot_threads_lock); |
239 | } | |
240 | ||
241 | static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu) | |
242 | { | |
243 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
244 | ||
7d7e499f | 245 | if (tsk && !ht->selfparking) |
f97f8f06 TG |
246 | kthread_park(tsk); |
247 | } | |
248 | ||
249 | void smpboot_park_threads(unsigned int cpu) | |
250 | { | |
251 | struct smp_hotplug_thread *cur; | |
252 | ||
253 | mutex_lock(&smpboot_threads_lock); | |
254 | list_for_each_entry_reverse(cur, &hotplug_threads, list) | |
255 | smpboot_park_thread(cur, cpu); | |
256 | mutex_unlock(&smpboot_threads_lock); | |
257 | } | |
258 | ||
259 | static void smpboot_destroy_threads(struct smp_hotplug_thread *ht) | |
260 | { | |
261 | unsigned int cpu; | |
262 | ||
263 | /* We need to destroy also the parked threads of offline cpus */ | |
264 | for_each_possible_cpu(cpu) { | |
265 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
266 | ||
267 | if (tsk) { | |
268 | kthread_stop(tsk); | |
269 | put_task_struct(tsk); | |
270 | *per_cpu_ptr(ht->store, cpu) = NULL; | |
271 | } | |
272 | } | |
273 | } | |
274 | ||
275 | /** | |
230ec939 FW |
276 | * smpboot_register_percpu_thread_cpumask - Register a per_cpu thread related |
277 | * to hotplug | |
f97f8f06 | 278 | * @plug_thread: Hotplug thread descriptor |
230ec939 | 279 | * @cpumask: The cpumask where threads run |
f97f8f06 TG |
280 | * |
281 | * Creates and starts the threads on all online cpus. | |
282 | */ | |
230ec939 FW |
283 | int smpboot_register_percpu_thread_cpumask(struct smp_hotplug_thread *plug_thread, |
284 | const struct cpumask *cpumask) | |
f97f8f06 TG |
285 | { |
286 | unsigned int cpu; | |
287 | int ret = 0; | |
288 | ||
b5242e98 CM |
289 | if (!alloc_cpumask_var(&plug_thread->cpumask, GFP_KERNEL)) |
290 | return -ENOMEM; | |
230ec939 | 291 | cpumask_copy(plug_thread->cpumask, cpumask); |
b5242e98 | 292 | |
4bee9686 | 293 | get_online_cpus(); |
f97f8f06 TG |
294 | mutex_lock(&smpboot_threads_lock); |
295 | for_each_online_cpu(cpu) { | |
296 | ret = __smpboot_create_thread(plug_thread, cpu); | |
297 | if (ret) { | |
298 | smpboot_destroy_threads(plug_thread); | |
5869b506 | 299 | free_cpumask_var(plug_thread->cpumask); |
f97f8f06 TG |
300 | goto out; |
301 | } | |
230ec939 FW |
302 | if (cpumask_test_cpu(cpu, cpumask)) |
303 | smpboot_unpark_thread(plug_thread, cpu); | |
f97f8f06 TG |
304 | } |
305 | list_add(&plug_thread->list, &hotplug_threads); | |
306 | out: | |
307 | mutex_unlock(&smpboot_threads_lock); | |
4bee9686 | 308 | put_online_cpus(); |
f97f8f06 TG |
309 | return ret; |
310 | } | |
230ec939 | 311 | EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread_cpumask); |
f97f8f06 TG |
312 | |
313 | /** | |
314 | * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug | |
315 | * @plug_thread: Hotplug thread descriptor | |
316 | * | |
317 | * Stops all threads on all possible cpus. | |
318 | */ | |
319 | void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread) | |
320 | { | |
321 | get_online_cpus(); | |
322 | mutex_lock(&smpboot_threads_lock); | |
323 | list_del(&plug_thread->list); | |
324 | smpboot_destroy_threads(plug_thread); | |
325 | mutex_unlock(&smpboot_threads_lock); | |
326 | put_online_cpus(); | |
b5242e98 | 327 | free_cpumask_var(plug_thread->cpumask); |
f97f8f06 TG |
328 | } |
329 | EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread); | |
8038dad7 | 330 | |
b5242e98 CM |
331 | /** |
332 | * smpboot_update_cpumask_percpu_thread - Adjust which per_cpu hotplug threads stay parked | |
333 | * @plug_thread: Hotplug thread descriptor | |
334 | * @new: Revised mask to use | |
335 | * | |
336 | * The cpumask field in the smp_hotplug_thread must not be updated directly | |
337 | * by the client, but only by calling this function. | |
fe4ba3c3 | 338 | * This function can only be called on a registered smp_hotplug_thread. |
b5242e98 CM |
339 | */ |
340 | int smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread, | |
341 | const struct cpumask *new) | |
342 | { | |
343 | struct cpumask *old = plug_thread->cpumask; | |
344 | cpumask_var_t tmp; | |
345 | unsigned int cpu; | |
346 | ||
347 | if (!alloc_cpumask_var(&tmp, GFP_KERNEL)) | |
348 | return -ENOMEM; | |
349 | ||
350 | get_online_cpus(); | |
351 | mutex_lock(&smpboot_threads_lock); | |
352 | ||
353 | /* Park threads that were exclusively enabled on the old mask. */ | |
354 | cpumask_andnot(tmp, old, new); | |
355 | for_each_cpu_and(cpu, tmp, cpu_online_mask) | |
356 | smpboot_park_thread(plug_thread, cpu); | |
357 | ||
358 | /* Unpark threads that are exclusively enabled on the new mask. */ | |
359 | cpumask_andnot(tmp, new, old); | |
360 | for_each_cpu_and(cpu, tmp, cpu_online_mask) | |
361 | smpboot_unpark_thread(plug_thread, cpu); | |
362 | ||
363 | cpumask_copy(old, new); | |
364 | ||
365 | mutex_unlock(&smpboot_threads_lock); | |
366 | put_online_cpus(); | |
367 | ||
368 | free_cpumask_var(tmp); | |
369 | ||
370 | return 0; | |
371 | } | |
372 | EXPORT_SYMBOL_GPL(smpboot_update_cpumask_percpu_thread); | |
373 | ||
8038dad7 PM |
374 | static DEFINE_PER_CPU(atomic_t, cpu_hotplug_state) = ATOMIC_INIT(CPU_POST_DEAD); |
375 | ||
376 | /* | |
377 | * Called to poll specified CPU's state, for example, when waiting for | |
378 | * a CPU to come online. | |
379 | */ | |
380 | int cpu_report_state(int cpu) | |
381 | { | |
382 | return atomic_read(&per_cpu(cpu_hotplug_state, cpu)); | |
383 | } | |
384 | ||
385 | /* | |
386 | * If CPU has died properly, set its state to CPU_UP_PREPARE and | |
387 | * return success. Otherwise, return -EBUSY if the CPU died after | |
388 | * cpu_wait_death() timed out. And yet otherwise again, return -EAGAIN | |
389 | * if cpu_wait_death() timed out and the CPU still hasn't gotten around | |
390 | * to dying. In the latter two cases, the CPU might not be set up | |
391 | * properly, but it is up to the arch-specific code to decide. | |
392 | * Finally, -EIO indicates an unanticipated problem. | |
393 | * | |
394 | * Note that it is permissible to omit this call entirely, as is | |
395 | * done in architectures that do no CPU-hotplug error checking. | |
396 | */ | |
397 | int cpu_check_up_prepare(int cpu) | |
398 | { | |
399 | if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) { | |
400 | atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE); | |
401 | return 0; | |
402 | } | |
403 | ||
404 | switch (atomic_read(&per_cpu(cpu_hotplug_state, cpu))) { | |
405 | ||
406 | case CPU_POST_DEAD: | |
407 | ||
408 | /* The CPU died properly, so just start it up again. */ | |
409 | atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE); | |
410 | return 0; | |
411 | ||
412 | case CPU_DEAD_FROZEN: | |
413 | ||
414 | /* | |
415 | * Timeout during CPU death, so let caller know. | |
416 | * The outgoing CPU completed its processing, but after | |
417 | * cpu_wait_death() timed out and reported the error. The | |
418 | * caller is free to proceed, in which case the state | |
419 | * will be reset properly by cpu_set_state_online(). | |
420 | * Proceeding despite this -EBUSY return makes sense | |
421 | * for systems where the outgoing CPUs take themselves | |
422 | * offline, with no post-death manipulation required from | |
423 | * a surviving CPU. | |
424 | */ | |
425 | return -EBUSY; | |
426 | ||
427 | case CPU_BROKEN: | |
428 | ||
429 | /* | |
430 | * The most likely reason we got here is that there was | |
431 | * a timeout during CPU death, and the outgoing CPU never | |
432 | * did complete its processing. This could happen on | |
433 | * a virtualized system if the outgoing VCPU gets preempted | |
434 | * for more than five seconds, and the user attempts to | |
435 | * immediately online that same CPU. Trying again later | |
436 | * might return -EBUSY above, hence -EAGAIN. | |
437 | */ | |
438 | return -EAGAIN; | |
439 | ||
440 | default: | |
441 | ||
442 | /* Should not happen. Famous last words. */ | |
443 | return -EIO; | |
444 | } | |
445 | } | |
446 | ||
447 | /* | |
448 | * Mark the specified CPU online. | |
449 | * | |
450 | * Note that it is permissible to omit this call entirely, as is | |
451 | * done in architectures that do no CPU-hotplug error checking. | |
452 | */ | |
453 | void cpu_set_state_online(int cpu) | |
454 | { | |
455 | (void)atomic_xchg(&per_cpu(cpu_hotplug_state, cpu), CPU_ONLINE); | |
456 | } | |
457 | ||
458 | #ifdef CONFIG_HOTPLUG_CPU | |
459 | ||
460 | /* | |
461 | * Wait for the specified CPU to exit the idle loop and die. | |
462 | */ | |
463 | bool cpu_wait_death(unsigned int cpu, int seconds) | |
464 | { | |
465 | int jf_left = seconds * HZ; | |
466 | int oldstate; | |
467 | bool ret = true; | |
468 | int sleep_jf = 1; | |
469 | ||
470 | might_sleep(); | |
471 | ||
472 | /* The outgoing CPU will normally get done quite quickly. */ | |
473 | if (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) == CPU_DEAD) | |
474 | goto update_state; | |
475 | udelay(5); | |
476 | ||
477 | /* But if the outgoing CPU dawdles, wait increasingly long times. */ | |
478 | while (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) != CPU_DEAD) { | |
479 | schedule_timeout_uninterruptible(sleep_jf); | |
480 | jf_left -= sleep_jf; | |
481 | if (jf_left <= 0) | |
482 | break; | |
483 | sleep_jf = DIV_ROUND_UP(sleep_jf * 11, 10); | |
484 | } | |
485 | update_state: | |
486 | oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu)); | |
487 | if (oldstate == CPU_DEAD) { | |
488 | /* Outgoing CPU died normally, update state. */ | |
489 | smp_mb(); /* atomic_read() before update. */ | |
490 | atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_POST_DEAD); | |
491 | } else { | |
492 | /* Outgoing CPU still hasn't died, set state accordingly. */ | |
493 | if (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), | |
494 | oldstate, CPU_BROKEN) != oldstate) | |
495 | goto update_state; | |
496 | ret = false; | |
497 | } | |
498 | return ret; | |
499 | } | |
500 | ||
501 | /* | |
502 | * Called by the outgoing CPU to report its successful death. Return | |
503 | * false if this report follows the surviving CPU's timing out. | |
504 | * | |
505 | * A separate "CPU_DEAD_FROZEN" is used when the surviving CPU | |
506 | * timed out. This approach allows architectures to omit calls to | |
507 | * cpu_check_up_prepare() and cpu_set_state_online() without defeating | |
508 | * the next cpu_wait_death()'s polling loop. | |
509 | */ | |
510 | bool cpu_report_death(void) | |
511 | { | |
512 | int oldstate; | |
513 | int newstate; | |
514 | int cpu = smp_processor_id(); | |
515 | ||
516 | do { | |
517 | oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu)); | |
518 | if (oldstate != CPU_BROKEN) | |
519 | newstate = CPU_DEAD; | |
520 | else | |
521 | newstate = CPU_DEAD_FROZEN; | |
522 | } while (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), | |
523 | oldstate, newstate) != oldstate); | |
524 | return newstate == CPU_DEAD; | |
525 | } | |
526 | ||
527 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ |