Commit | Line | Data |
---|---|---|
ffdb5976 | 1 | /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
e5582ca2 RR |
2 | * GPL v2 and any later version. |
3 | */ | |
1da177e4 LT |
4 | #include <linux/cpu.h> |
5 | #include <linux/err.h> | |
ee527cd3 PB |
6 | #include <linux/kthread.h> |
7 | #include <linux/module.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/stop_machine.h> | |
1da177e4 | 10 | #include <linux/syscalls.h> |
a12bb444 BH |
11 | #include <linux/interrupt.h> |
12 | ||
1da177e4 | 13 | #include <asm/atomic.h> |
1da177e4 LT |
14 | #include <asm/uaccess.h> |
15 | ||
ffdb5976 | 16 | /* This controls the threads on each CPU. */ |
1da177e4 | 17 | enum stopmachine_state { |
ffdb5976 RR |
18 | /* Dummy starting state for thread. */ |
19 | STOPMACHINE_NONE, | |
20 | /* Awaiting everyone to be scheduled. */ | |
1da177e4 | 21 | STOPMACHINE_PREPARE, |
ffdb5976 | 22 | /* Disable interrupts. */ |
1da177e4 | 23 | STOPMACHINE_DISABLE_IRQ, |
ffdb5976 | 24 | /* Run the function */ |
5c2aed62 | 25 | STOPMACHINE_RUN, |
ffdb5976 | 26 | /* Exit */ |
1da177e4 LT |
27 | STOPMACHINE_EXIT, |
28 | }; | |
ffdb5976 | 29 | static enum stopmachine_state state; |
1da177e4 | 30 | |
5c2aed62 JB |
31 | struct stop_machine_data { |
32 | int (*fn)(void *); | |
33 | void *data; | |
ffdb5976 RR |
34 | int fnret; |
35 | }; | |
5c2aed62 | 36 | |
ffdb5976 RR |
37 | /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ |
38 | static unsigned int num_threads; | |
39 | static atomic_t thread_ack; | |
ffdb5976 | 40 | static DEFINE_MUTEX(lock); |
1da177e4 | 41 | |
c9583e55 HC |
42 | static struct workqueue_struct *stop_machine_wq; |
43 | static struct stop_machine_data active, idle; | |
44 | static const cpumask_t *active_cpus; | |
45 | static void *stop_machine_work; | |
46 | ||
ffdb5976 | 47 | static void set_state(enum stopmachine_state newstate) |
1da177e4 | 48 | { |
ffdb5976 RR |
49 | /* Reset ack counter. */ |
50 | atomic_set(&thread_ack, num_threads); | |
51 | smp_wmb(); | |
52 | state = newstate; | |
1da177e4 LT |
53 | } |
54 | ||
ffdb5976 RR |
55 | /* Last one to ack a state moves to the next state. */ |
56 | static void ack_state(void) | |
1da177e4 | 57 | { |
c9583e55 HC |
58 | if (atomic_dec_and_test(&thread_ack)) |
59 | set_state(state + 1); | |
1da177e4 LT |
60 | } |
61 | ||
c9583e55 HC |
62 | /* This is the actual function which stops the CPU. It runs |
63 | * in the context of a dedicated stopmachine workqueue. */ | |
64 | static void stop_cpu(struct work_struct *unused) | |
1da177e4 | 65 | { |
ffdb5976 | 66 | enum stopmachine_state curstate = STOPMACHINE_NONE; |
c9583e55 HC |
67 | struct stop_machine_data *smdata = &idle; |
68 | int cpu = smp_processor_id(); | |
8163bcac | 69 | int err; |
c9583e55 HC |
70 | |
71 | if (!active_cpus) { | |
72 | if (cpu == first_cpu(cpu_online_map)) | |
73 | smdata = &active; | |
74 | } else { | |
75 | if (cpu_isset(cpu, *active_cpus)) | |
76 | smdata = &active; | |
77 | } | |
ffdb5976 RR |
78 | /* Simple state machine */ |
79 | do { | |
80 | /* Chill out and ensure we re-read stopmachine_state. */ | |
3401a61e | 81 | cpu_relax(); |
ffdb5976 RR |
82 | if (state != curstate) { |
83 | curstate = state; | |
84 | switch (curstate) { | |
85 | case STOPMACHINE_DISABLE_IRQ: | |
86 | local_irq_disable(); | |
87 | hard_irq_disable(); | |
88 | break; | |
89 | case STOPMACHINE_RUN: | |
8163bcac HC |
90 | /* On multiple CPUs only a single error code |
91 | * is needed to tell that something failed. */ | |
92 | err = smdata->fn(smdata->data); | |
93 | if (err) | |
94 | smdata->fnret = err; | |
ffdb5976 RR |
95 | break; |
96 | default: | |
97 | break; | |
98 | } | |
99 | ack_state(); | |
100 | } | |
101 | } while (curstate != STOPMACHINE_EXIT); | |
1da177e4 | 102 | |
1da177e4 LT |
103 | local_irq_enable(); |
104 | } | |
105 | ||
ffdb5976 RR |
106 | /* Callback for CPUs which aren't supposed to do anything. */ |
107 | static int chill(void *unused) | |
5c2aed62 | 108 | { |
ffdb5976 | 109 | return 0; |
5c2aed62 | 110 | } |
1da177e4 | 111 | |
eeec4fad | 112 | int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
1da177e4 | 113 | { |
c9583e55 | 114 | struct work_struct *sm_work; |
e14c8bf8 | 115 | int i, ret; |
ffdb5976 | 116 | |
c9583e55 HC |
117 | /* Set up initial state. */ |
118 | mutex_lock(&lock); | |
119 | num_threads = num_online_cpus(); | |
120 | active_cpus = cpus; | |
ffdb5976 RR |
121 | active.fn = fn; |
122 | active.data = data; | |
123 | active.fnret = 0; | |
124 | idle.fn = chill; | |
125 | idle.data = NULL; | |
126 | ||
ffdb5976 | 127 | set_state(STOPMACHINE_PREPARE); |
1da177e4 | 128 | |
c9583e55 | 129 | /* Schedule the stop_cpu work on all cpus: hold this CPU so one |
ffdb5976 | 130 | * doesn't hit this CPU until we're ready. */ |
eeec4fad | 131 | get_cpu(); |
c9583e55 HC |
132 | for_each_online_cpu(i) { |
133 | sm_work = percpu_ptr(stop_machine_work, i); | |
134 | INIT_WORK(sm_work, stop_cpu); | |
135 | queue_work_on(i, stop_machine_wq, sm_work); | |
136 | } | |
ffdb5976 RR |
137 | /* This will release the thread on our CPU. */ |
138 | put_cpu(); | |
c9583e55 | 139 | flush_workqueue(stop_machine_wq); |
e14c8bf8 | 140 | ret = active.fnret; |
ffdb5976 | 141 | mutex_unlock(&lock); |
e14c8bf8 | 142 | return ret; |
1da177e4 LT |
143 | } |
144 | ||
eeec4fad | 145 | int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
1da177e4 | 146 | { |
1da177e4 LT |
147 | int ret; |
148 | ||
149 | /* No CPUs can come up or down during this. */ | |
86ef5c9a | 150 | get_online_cpus(); |
eeec4fad | 151 | ret = __stop_machine(fn, data, cpus); |
86ef5c9a | 152 | put_online_cpus(); |
1da177e4 LT |
153 | |
154 | return ret; | |
155 | } | |
eeec4fad | 156 | EXPORT_SYMBOL_GPL(stop_machine); |
c9583e55 HC |
157 | |
158 | static int __init stop_machine_init(void) | |
159 | { | |
160 | stop_machine_wq = create_rt_workqueue("kstop"); | |
161 | stop_machine_work = alloc_percpu(struct work_struct); | |
162 | return 0; | |
163 | } | |
4403b406 | 164 | core_initcall(stop_machine_init); |