Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * SMP Support | |
3 | * | |
4 | * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> | |
5 | * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com> | |
6 | * | |
7 | * Lots of stuff stolen from arch/alpha/kernel/smp.c | |
8 | * | |
9 | * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized | |
10 | * the existing code (on the lines of x86 port). | |
11 | * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy | |
12 | * calibration on each CPU. | |
13 | * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id | |
14 | * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor | |
15 | * & cpu_online_map now gets done here (instead of setup.c) | |
16 | * 99/10/05 davidm Update to bring it in sync with new command-line processing | |
17 | * scheme. | |
18 | * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and | |
19 | * smp_call_function_single to resend IPI on timeouts | |
20 | */ | |
21 | #include <linux/module.h> | |
22 | #include <linux/kernel.h> | |
23 | #include <linux/sched.h> | |
24 | #include <linux/init.h> | |
25 | #include <linux/interrupt.h> | |
26 | #include <linux/smp.h> | |
27 | #include <linux/kernel_stat.h> | |
28 | #include <linux/mm.h> | |
29 | #include <linux/cache.h> | |
30 | #include <linux/delay.h> | |
31 | #include <linux/efi.h> | |
32 | #include <linux/bitops.h> | |
a7956113 | 33 | #include <linux/kexec.h> |
1da177e4 | 34 | |
60063497 | 35 | #include <linux/atomic.h> |
1da177e4 LT |
36 | #include <asm/current.h> |
37 | #include <asm/delay.h> | |
38 | #include <asm/machvec.h> | |
39 | #include <asm/io.h> | |
40 | #include <asm/irq.h> | |
41 | #include <asm/page.h> | |
42 | #include <asm/pgalloc.h> | |
43 | #include <asm/pgtable.h> | |
44 | #include <asm/processor.h> | |
45 | #include <asm/ptrace.h> | |
46 | #include <asm/sal.h> | |
1da177e4 LT |
47 | #include <asm/tlbflush.h> |
48 | #include <asm/unistd.h> | |
49 | #include <asm/mca.h> | |
50 | ||
3be44b9c JS |
51 | /* |
52 | * Note: alignment of 4 entries/cacheline was empirically determined | |
53 | * to be a good tradeoff between hot cachelines & spreading the array | |
54 | * across too many cacheline. | |
55 | */ | |
56 | static struct local_tlb_flush_counts { | |
57 | unsigned int count; | |
58 | } __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS]; | |
59 | ||
b9bf3121 TH |
60 | static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned short [NR_CPUS], |
61 | shadow_flush_counts); | |
3be44b9c | 62 | |
1da177e4 LT |
63 | #define IPI_CALL_FUNC 0 |
64 | #define IPI_CPU_STOP 1 | |
f27b433e | 65 | #define IPI_CALL_FUNC_SINGLE 2 |
a7956113 | 66 | #define IPI_KDUMP_CPU_STOP 3 |
1da177e4 LT |
67 | |
68 | /* This needs to be cacheline aligned because it is written to by *other* CPUs. */ | |
e088a4ad | 69 | static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, ipi_operation); |
1da177e4 LT |
70 | |
71 | extern void cpu_halt (void); | |
72 | ||
1da177e4 | 73 | static void |
c0cd661b | 74 | stop_this_cpu(void) |
1da177e4 LT |
75 | { |
76 | /* | |
77 | * Remove this CPU: | |
78 | */ | |
7d7f9848 | 79 | set_cpu_online(smp_processor_id(), false); |
1da177e4 LT |
80 | max_xtp(); |
81 | local_irq_disable(); | |
82 | cpu_halt(); | |
83 | } | |
84 | ||
85 | void | |
86 | cpu_die(void) | |
87 | { | |
88 | max_xtp(); | |
89 | local_irq_disable(); | |
90 | cpu_halt(); | |
91 | /* Should never be here */ | |
92 | BUG(); | |
93 | for (;;); | |
94 | } | |
95 | ||
96 | irqreturn_t | |
024e4f2c | 97 | handle_IPI (int irq, void *dev_id) |
1da177e4 LT |
98 | { |
99 | int this_cpu = get_cpu(); | |
100 | unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation); | |
101 | unsigned long ops; | |
102 | ||
103 | mb(); /* Order interrupt and bit testing. */ | |
104 | while ((ops = xchg(pending_ipis, 0)) != 0) { | |
105 | mb(); /* Order bit clearing and data access. */ | |
106 | do { | |
107 | unsigned long which; | |
108 | ||
109 | which = ffz(~ops); | |
110 | ops &= ~(1 << which); | |
111 | ||
112 | switch (which) { | |
c0cd661b | 113 | case IPI_CPU_STOP: |
1da177e4 LT |
114 | stop_this_cpu(); |
115 | break; | |
f27b433e JA |
116 | case IPI_CALL_FUNC: |
117 | generic_smp_call_function_interrupt(); | |
118 | break; | |
119 | case IPI_CALL_FUNC_SINGLE: | |
120 | generic_smp_call_function_single_interrupt(); | |
121 | break; | |
45a98fc6 | 122 | #ifdef CONFIG_KEXEC |
c0cd661b | 123 | case IPI_KDUMP_CPU_STOP: |
a7956113 ZN |
124 | unw_init_running(kdump_cpu_freeze, NULL); |
125 | break; | |
126 | #endif | |
c0cd661b HS |
127 | default: |
128 | printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", | |
129 | this_cpu, which); | |
1da177e4 LT |
130 | break; |
131 | } | |
132 | } while (ops); | |
133 | mb(); /* Order data access and bit testing. */ | |
134 | } | |
135 | put_cpu(); | |
136 | return IRQ_HANDLED; | |
137 | } | |
138 | ||
f27b433e JA |
139 | |
140 | ||
1da177e4 | 141 | /* |
72fdbdce | 142 | * Called with preemption disabled. |
1da177e4 LT |
143 | */ |
144 | static inline void | |
145 | send_IPI_single (int dest_cpu, int op) | |
146 | { | |
147 | set_bit(op, &per_cpu(ipi_operation, dest_cpu)); | |
148 | platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0); | |
149 | } | |
150 | ||
151 | /* | |
72fdbdce | 152 | * Called with preemption disabled. |
1da177e4 LT |
153 | */ |
154 | static inline void | |
155 | send_IPI_allbutself (int op) | |
156 | { | |
157 | unsigned int i; | |
158 | ||
dc565b52 | 159 | for_each_online_cpu(i) { |
160 | if (i != smp_processor_id()) | |
1da177e4 LT |
161 | send_IPI_single(i, op); |
162 | } | |
163 | } | |
164 | ||
31a6b11f XZ |
165 | /* |
166 | * Called with preemption disabled. | |
167 | */ | |
168 | static inline void | |
40fe697a | 169 | send_IPI_mask(const struct cpumask *mask, int op) |
31a6b11f XZ |
170 | { |
171 | unsigned int cpu; | |
172 | ||
40fe697a | 173 | for_each_cpu(cpu, mask) { |
31a6b11f XZ |
174 | send_IPI_single(cpu, op); |
175 | } | |
176 | } | |
177 | ||
1da177e4 | 178 | /* |
72fdbdce | 179 | * Called with preemption disabled. |
1da177e4 LT |
180 | */ |
181 | static inline void | |
182 | send_IPI_all (int op) | |
183 | { | |
184 | int i; | |
185 | ||
dc565b52 | 186 | for_each_online_cpu(i) { |
187 | send_IPI_single(i, op); | |
188 | } | |
1da177e4 LT |
189 | } |
190 | ||
191 | /* | |
72fdbdce | 192 | * Called with preemption disabled. |
1da177e4 LT |
193 | */ |
194 | static inline void | |
195 | send_IPI_self (int op) | |
196 | { | |
197 | send_IPI_single(smp_processor_id(), op); | |
198 | } | |
199 | ||
45a98fc6 | 200 | #ifdef CONFIG_KEXEC |
a7956113 | 201 | void |
ccbebdac | 202 | kdump_smp_send_stop(void) |
a7956113 ZN |
203 | { |
204 | send_IPI_allbutself(IPI_KDUMP_CPU_STOP); | |
205 | } | |
206 | ||
207 | void | |
ccbebdac | 208 | kdump_smp_send_init(void) |
a7956113 ZN |
209 | { |
210 | unsigned int cpu, self_cpu; | |
211 | self_cpu = smp_processor_id(); | |
212 | for_each_online_cpu(cpu) { | |
213 | if (cpu != self_cpu) { | |
214 | if(kdump_status[cpu] == 0) | |
215 | platform_send_ipi(cpu, 0, IA64_IPI_DM_INIT, 0); | |
216 | } | |
217 | } | |
218 | } | |
219 | #endif | |
1da177e4 | 220 | /* |
72fdbdce | 221 | * Called with preemption disabled. |
1da177e4 LT |
222 | */ |
223 | void | |
224 | smp_send_reschedule (int cpu) | |
225 | { | |
226 | platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0); | |
227 | } | |
c4cb768f | 228 | EXPORT_SYMBOL_GPL(smp_send_reschedule); |
1da177e4 | 229 | |
3be44b9c | 230 | /* |
72fdbdce | 231 | * Called with preemption disabled. |
3be44b9c JS |
232 | */ |
233 | static void | |
234 | smp_send_local_flush_tlb (int cpu) | |
235 | { | |
236 | platform_send_ipi(cpu, IA64_IPI_LOCAL_TLB_FLUSH, IA64_IPI_DM_INT, 0); | |
237 | } | |
238 | ||
239 | void | |
240 | smp_local_flush_tlb(void) | |
241 | { | |
242 | /* | |
243 | * Use atomic ops. Otherwise, the load/increment/store sequence from | |
244 | * a "++" operation can have the line stolen between the load & store. | |
245 | * The overhead of the atomic op in negligible in this case & offers | |
246 | * significant benefit for the brief periods where lots of cpus | |
247 | * are simultaneously flushing TLBs. | |
248 | */ | |
249 | ia64_fetchadd(1, &local_tlb_flush_counts[smp_processor_id()].count, acq); | |
250 | local_flush_tlb_all(); | |
251 | } | |
252 | ||
253 | #define FLUSH_DELAY 5 /* Usec backoff to eliminate excessive cacheline bouncing */ | |
254 | ||
255 | void | |
256 | smp_flush_tlb_cpumask(cpumask_t xcpumask) | |
257 | { | |
97653f92 | 258 | unsigned short *counts = __ia64_per_cpu_var(shadow_flush_counts); |
3be44b9c JS |
259 | cpumask_t cpumask = xcpumask; |
260 | int mycpu, cpu, flush_mycpu = 0; | |
261 | ||
262 | preempt_disable(); | |
263 | mycpu = smp_processor_id(); | |
264 | ||
5d2068da | 265 | for_each_cpu(cpu, &cpumask) |
97653f92 | 266 | counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff; |
3be44b9c JS |
267 | |
268 | mb(); | |
5d2068da | 269 | for_each_cpu(cpu, &cpumask) { |
3be44b9c JS |
270 | if (cpu == mycpu) |
271 | flush_mycpu = 1; | |
272 | else | |
273 | smp_send_local_flush_tlb(cpu); | |
274 | } | |
275 | ||
276 | if (flush_mycpu) | |
277 | smp_local_flush_tlb(); | |
278 | ||
5d2068da | 279 | for_each_cpu(cpu, &cpumask) |
97653f92 | 280 | while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff)) |
3be44b9c JS |
281 | udelay(FLUSH_DELAY); |
282 | ||
283 | preempt_enable(); | |
284 | } | |
285 | ||
1da177e4 LT |
286 | void |
287 | smp_flush_tlb_all (void) | |
288 | { | |
15c8b6c1 | 289 | on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1); |
1da177e4 LT |
290 | } |
291 | ||
292 | void | |
293 | smp_flush_tlb_mm (struct mm_struct *mm) | |
294 | { | |
75c1c91c | 295 | cpumask_var_t cpus; |
a68db763 | 296 | preempt_disable(); |
1da177e4 LT |
297 | /* this happens for the common case of a single-threaded fork(): */ |
298 | if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1)) | |
299 | { | |
300 | local_finish_flush_tlb_mm(mm); | |
a68db763 | 301 | preempt_enable(); |
1da177e4 LT |
302 | return; |
303 | } | |
75c1c91c DS |
304 | if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) { |
305 | smp_call_function((void (*)(void *))local_finish_flush_tlb_mm, | |
306 | mm, 1); | |
307 | } else { | |
308 | cpumask_copy(cpus, mm_cpumask(mm)); | |
309 | smp_call_function_many(cpus, | |
310 | (void (*)(void *))local_finish_flush_tlb_mm, mm, 1); | |
311 | free_cpumask_var(cpus); | |
312 | } | |
edb91dc0 DS |
313 | local_irq_disable(); |
314 | local_finish_flush_tlb_mm(mm); | |
315 | local_irq_enable(); | |
a68db763 | 316 | preempt_enable(); |
1da177e4 LT |
317 | } |
318 | ||
f27b433e | 319 | void arch_send_call_function_single_ipi(int cpu) |
31a6b11f | 320 | { |
f27b433e | 321 | send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE); |
31a6b11f | 322 | } |
31a6b11f | 323 | |
40fe697a | 324 | void arch_send_call_function_ipi_mask(const struct cpumask *mask) |
1da177e4 | 325 | { |
f27b433e | 326 | send_IPI_mask(mask, IPI_CALL_FUNC); |
1da177e4 | 327 | } |
1da177e4 LT |
328 | |
329 | /* | |
330 | * this function calls the 'stop' function on all other CPUs in the system. | |
331 | */ | |
332 | void | |
333 | smp_send_stop (void) | |
334 | { | |
335 | send_IPI_allbutself(IPI_CPU_STOP); | |
336 | } | |
337 | ||
cb2e0912 | 338 | int |
1da177e4 LT |
339 | setup_profiling_timer (unsigned int multiplier) |
340 | { | |
341 | return -EINVAL; | |
342 | } |