Commit | Line | Data |
---|---|---|
1812924b CW |
1 | /* |
2 | * SGI UltraViolet TLB flush routines. | |
3 | * | |
4 | * (c) 2008 Cliff Wickman <cpw@sgi.com>, SGI. | |
5 | * | |
6 | * This code is released under the GNU General Public License version 2 or | |
7 | * later. | |
8 | */ | |
9 | #include <linux/mc146818rtc.h> | |
10 | #include <linux/proc_fs.h> | |
11 | #include <linux/kernel.h> | |
12 | ||
1812924b | 13 | #include <asm/mmu_context.h> |
1812924b | 14 | #include <asm/uv/uv_mmrs.h> |
b4c286e6 | 15 | #include <asm/uv/uv_hub.h> |
1812924b | 16 | #include <asm/uv/uv_bau.h> |
b4c286e6 IM |
17 | #include <asm/genapic.h> |
18 | #include <asm/idle.h> | |
b194b120 | 19 | #include <asm/tsc.h> |
99dd8713 | 20 | #include <asm/irq_vectors.h> |
1812924b | 21 | |
b194b120 CW |
22 | #include <mach_apic.h> |
23 | ||
b4c286e6 IM |
24 | static struct bau_control **uv_bau_table_bases __read_mostly; |
25 | static int uv_bau_retry_limit __read_mostly; | |
26 | ||
27 | /* position of pnode (which is nasid>>1): */ | |
28 | static int uv_nshift __read_mostly; | |
29 | ||
30 | static unsigned long uv_mmask __read_mostly; | |
1812924b | 31 | |
dc163a41 IM |
32 | static DEFINE_PER_CPU(struct ptc_stats, ptcstats); |
33 | static DEFINE_PER_CPU(struct bau_control, bau_control); | |
1812924b CW |
34 | |
35 | /* | |
36 | * Free a software acknowledge hardware resource by clearing its Pending | |
37 | * bit. This will return a reply to the sender. | |
38 | * If the message has timed out, a reply has already been sent by the | |
39 | * hardware but the resource has not been released. In that case our | |
40 | * clear of the Timeout bit (as well) will free the resource. No reply will | |
41 | * be sent (the hardware will only do one reply per message). | |
42 | */ | |
b194b120 | 43 | static void uv_reply_to_message(int resource, |
b4c286e6 IM |
44 | struct bau_payload_queue_entry *msg, |
45 | struct bau_msg_status *msp) | |
1812924b | 46 | { |
b194b120 | 47 | unsigned long dw; |
1812924b | 48 | |
b194b120 | 49 | dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource); |
1812924b CW |
50 | msg->replied_to = 1; |
51 | msg->sw_ack_vector = 0; | |
52 | if (msp) | |
53 | msp->seen_by.bits = 0; | |
b194b120 | 54 | uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw); |
1812924b CW |
55 | } |
56 | ||
57 | /* | |
58 | * Do all the things a cpu should do for a TLB shootdown message. | |
59 | * Other cpu's may come here at the same time for this message. | |
60 | */ | |
b194b120 | 61 | static void uv_bau_process_message(struct bau_payload_queue_entry *msg, |
b4c286e6 | 62 | int msg_slot, int sw_ack_slot) |
1812924b | 63 | { |
1812924b CW |
64 | unsigned long this_cpu_mask; |
65 | struct bau_msg_status *msp; | |
b4c286e6 | 66 | int cpu; |
1812924b CW |
67 | |
68 | msp = __get_cpu_var(bau_control).msg_statuses + msg_slot; | |
69 | cpu = uv_blade_processor_id(); | |
70 | msg->number_of_cpus = | |
71 | uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id())); | |
dc163a41 | 72 | this_cpu_mask = 1UL << cpu; |
1812924b CW |
73 | if (msp->seen_by.bits & this_cpu_mask) |
74 | return; | |
75 | atomic_or_long(&msp->seen_by.bits, this_cpu_mask); | |
76 | ||
77 | if (msg->replied_to == 1) | |
78 | return; | |
79 | ||
80 | if (msg->address == TLB_FLUSH_ALL) { | |
81 | local_flush_tlb(); | |
82 | __get_cpu_var(ptcstats).alltlb++; | |
83 | } else { | |
84 | __flush_tlb_one(msg->address); | |
85 | __get_cpu_var(ptcstats).onetlb++; | |
86 | } | |
87 | ||
88 | __get_cpu_var(ptcstats).requestee++; | |
89 | ||
90 | atomic_inc_short(&msg->acknowledge_count); | |
91 | if (msg->number_of_cpus == msg->acknowledge_count) | |
92 | uv_reply_to_message(sw_ack_slot, msg, msp); | |
1812924b CW |
93 | } |
94 | ||
95 | /* | |
dc163a41 | 96 | * Examine the payload queue on one distribution node to see |
1812924b CW |
97 | * which messages have not been seen, and which cpu(s) have not seen them. |
98 | * | |
99 | * Returns the number of cpu's that have not responded. | |
100 | */ | |
dc163a41 | 101 | static int uv_examine_destination(struct bau_control *bau_tablesp, int sender) |
1812924b | 102 | { |
1812924b CW |
103 | struct bau_payload_queue_entry *msg; |
104 | struct bau_msg_status *msp; | |
b4c286e6 IM |
105 | int count = 0; |
106 | int i; | |
107 | int j; | |
1812924b | 108 | |
dc163a41 IM |
109 | for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE; |
110 | msg++, i++) { | |
111 | if ((msg->sending_cpu == sender) && (!msg->replied_to)) { | |
112 | msp = bau_tablesp->msg_statuses + i; | |
113 | printk(KERN_DEBUG | |
114 | "blade %d: address:%#lx %d of %d, not cpu(s): ", | |
115 | i, msg->address, msg->acknowledge_count, | |
116 | msg->number_of_cpus); | |
117 | for (j = 0; j < msg->number_of_cpus; j++) { | |
b4c286e6 | 118 | if (!((1L << j) & msp->seen_by.bits)) { |
dc163a41 IM |
119 | count++; |
120 | printk("%d ", j); | |
121 | } | |
122 | } | |
123 | printk("\n"); | |
124 | } | |
125 | } | |
126 | return count; | |
127 | } | |
128 | ||
129 | /* | |
130 | * Examine the payload queue on all the distribution nodes to see | |
131 | * which messages have not been seen, and which cpu(s) have not seen them. | |
132 | * | |
133 | * Returns the number of cpu's that have not responded. | |
134 | */ | |
135 | static int uv_examine_destinations(struct bau_target_nodemask *distribution) | |
136 | { | |
137 | int sender; | |
138 | int i; | |
139 | int count = 0; | |
140 | ||
1812924b | 141 | sender = smp_processor_id(); |
b4c286e6 | 142 | for (i = 0; i < sizeof(struct bau_target_nodemask) * BITSPERBYTE; i++) { |
b194b120 CW |
143 | if (!bau_node_isset(i, distribution)) |
144 | continue; | |
dc163a41 | 145 | count += uv_examine_destination(uv_bau_table_bases[i], sender); |
1812924b CW |
146 | } |
147 | return count; | |
148 | } | |
149 | ||
b194b120 CW |
150 | /* |
151 | * wait for completion of a broadcast message | |
152 | * | |
153 | * return COMPLETE, RETRY or GIVEUP | |
154 | */ | |
dc163a41 | 155 | static int uv_wait_completion(struct bau_desc *bau_desc, |
b194b120 CW |
156 | unsigned long mmr_offset, int right_shift) |
157 | { | |
158 | int exams = 0; | |
159 | long destination_timeouts = 0; | |
160 | long source_timeouts = 0; | |
161 | unsigned long descriptor_status; | |
162 | ||
163 | while ((descriptor_status = (((unsigned long) | |
164 | uv_read_local_mmr(mmr_offset) >> | |
165 | right_shift) & UV_ACT_STATUS_MASK)) != | |
166 | DESC_STATUS_IDLE) { | |
167 | if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) { | |
168 | source_timeouts++; | |
169 | if (source_timeouts > SOURCE_TIMEOUT_LIMIT) | |
170 | source_timeouts = 0; | |
171 | __get_cpu_var(ptcstats).s_retry++; | |
172 | return FLUSH_RETRY; | |
173 | } | |
174 | /* | |
175 | * spin here looking for progress at the destinations | |
176 | */ | |
177 | if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) { | |
178 | destination_timeouts++; | |
179 | if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) { | |
180 | /* | |
181 | * returns number of cpus not responding | |
182 | */ | |
183 | if (uv_examine_destinations | |
184 | (&bau_desc->distribution) == 0) { | |
185 | __get_cpu_var(ptcstats).d_retry++; | |
186 | return FLUSH_RETRY; | |
187 | } | |
188 | exams++; | |
189 | if (exams >= uv_bau_retry_limit) { | |
190 | printk(KERN_DEBUG | |
191 | "uv_flush_tlb_others"); | |
192 | printk("giving up on cpu %d\n", | |
193 | smp_processor_id()); | |
194 | return FLUSH_GIVEUP; | |
195 | } | |
196 | /* | |
197 | * delays can hang the simulator | |
198 | udelay(1000); | |
199 | */ | |
200 | destination_timeouts = 0; | |
201 | } | |
202 | } | |
203 | } | |
204 | return FLUSH_COMPLETE; | |
205 | } | |
206 | ||
207 | /** | |
208 | * uv_flush_send_and_wait | |
209 | * | |
210 | * Send a broadcast and wait for a broadcast message to complete. | |
211 | * | |
212 | * The cpumaskp mask contains the cpus the broadcast was sent to. | |
213 | * | |
214 | * Returns 1 if all remote flushing was done. The mask is zeroed. | |
215 | * Returns 0 if some remote flushing remains to be done. The mask is left | |
216 | * unchanged. | |
217 | */ | |
dc163a41 IM |
218 | int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc, |
219 | cpumask_t *cpumaskp) | |
b194b120 CW |
220 | { |
221 | int completion_status = 0; | |
222 | int right_shift; | |
b194b120 | 223 | int tries = 0; |
b4c286e6 IM |
224 | int blade; |
225 | int bit; | |
b194b120 | 226 | unsigned long mmr_offset; |
b4c286e6 | 227 | unsigned long index; |
b194b120 CW |
228 | cycles_t time1; |
229 | cycles_t time2; | |
230 | ||
231 | if (cpu < UV_CPUS_PER_ACT_STATUS) { | |
232 | mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0; | |
233 | right_shift = cpu * UV_ACT_STATUS_SIZE; | |
234 | } else { | |
235 | mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1; | |
236 | right_shift = | |
237 | ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE); | |
238 | } | |
239 | time1 = get_cycles(); | |
240 | do { | |
241 | tries++; | |
dc163a41 IM |
242 | index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | |
243 | cpu; | |
b194b120 CW |
244 | uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index); |
245 | completion_status = uv_wait_completion(bau_desc, mmr_offset, | |
246 | right_shift); | |
247 | } while (completion_status == FLUSH_RETRY); | |
248 | time2 = get_cycles(); | |
249 | __get_cpu_var(ptcstats).sflush += (time2 - time1); | |
250 | if (tries > 1) | |
251 | __get_cpu_var(ptcstats).retriesok++; | |
252 | ||
253 | if (completion_status == FLUSH_GIVEUP) { | |
254 | /* | |
255 | * Cause the caller to do an IPI-style TLB shootdown on | |
256 | * the cpu's, all of which are still in the mask. | |
257 | */ | |
258 | __get_cpu_var(ptcstats).ptc_i++; | |
259 | return 0; | |
260 | } | |
261 | ||
262 | /* | |
263 | * Success, so clear the remote cpu's from the mask so we don't | |
264 | * use the IPI method of shootdown on them. | |
265 | */ | |
266 | for_each_cpu_mask(bit, *cpumaskp) { | |
267 | blade = uv_cpu_to_blade_id(bit); | |
268 | if (blade == this_blade) | |
269 | continue; | |
270 | cpu_clear(bit, *cpumaskp); | |
271 | } | |
272 | if (!cpus_empty(*cpumaskp)) | |
273 | return 0; | |
274 | return 1; | |
275 | } | |
276 | ||
1812924b CW |
277 | /** |
278 | * uv_flush_tlb_others - globally purge translation cache of a virtual | |
279 | * address or all TLB's | |
280 | * @cpumaskp: mask of all cpu's in which the address is to be removed | |
281 | * @mm: mm_struct containing virtual address range | |
282 | * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu) | |
283 | * | |
284 | * This is the entry point for initiating any UV global TLB shootdown. | |
285 | * | |
286 | * Purges the translation caches of all specified processors of the given | |
287 | * virtual address, or purges all TLB's on specified processors. | |
288 | * | |
289 | * The caller has derived the cpumaskp from the mm_struct and has subtracted | |
290 | * the local cpu from the mask. This function is called only if there | |
291 | * are bits set in the mask. (e.g. flush_tlb_page()) | |
292 | * | |
293 | * The cpumaskp is converted into a nodemask of the nodes containing | |
294 | * the cpus. | |
b194b120 CW |
295 | * |
296 | * Returns 1 if all remote flushing was done. | |
297 | * Returns 0 if some remote flushing remains to be done. | |
1812924b | 298 | */ |
b194b120 | 299 | int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, |
b4c286e6 | 300 | unsigned long va) |
1812924b CW |
301 | { |
302 | int i; | |
b194b120 | 303 | int bit; |
1812924b CW |
304 | int blade; |
305 | int cpu; | |
1812924b | 306 | int this_blade; |
b194b120 | 307 | int locals = 0; |
dc163a41 | 308 | struct bau_desc *bau_desc; |
1812924b CW |
309 | |
310 | cpu = uv_blade_processor_id(); | |
311 | this_blade = uv_numa_blade_id(); | |
312 | bau_desc = __get_cpu_var(bau_control).descriptor_base; | |
b194b120 | 313 | bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu; |
1812924b CW |
314 | |
315 | bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); | |
316 | ||
317 | i = 0; | |
318 | for_each_cpu_mask(bit, *cpumaskp) { | |
319 | blade = uv_cpu_to_blade_id(bit); | |
dc163a41 | 320 | BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1)); |
b194b120 CW |
321 | if (blade == this_blade) { |
322 | locals++; | |
1812924b | 323 | continue; |
b194b120 | 324 | } |
1812924b | 325 | bau_node_set(blade, &bau_desc->distribution); |
1812924b CW |
326 | i++; |
327 | } | |
b194b120 CW |
328 | if (i == 0) { |
329 | /* | |
330 | * no off_node flushing; return status for local node | |
331 | */ | |
332 | if (locals) | |
333 | return 0; | |
334 | else | |
335 | return 1; | |
336 | } | |
1812924b CW |
337 | __get_cpu_var(ptcstats).requestor++; |
338 | __get_cpu_var(ptcstats).ntargeted += i; | |
339 | ||
340 | bau_desc->payload.address = va; | |
341 | bau_desc->payload.sending_cpu = smp_processor_id(); | |
342 | ||
b194b120 | 343 | return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp); |
1812924b CW |
344 | } |
345 | ||
346 | /* | |
347 | * The BAU message interrupt comes here. (registered by set_intr_gate) | |
348 | * See entry_64.S | |
349 | * | |
350 | * We received a broadcast assist message. | |
351 | * | |
352 | * Interrupts may have been disabled; this interrupt could represent | |
353 | * the receipt of several messages. | |
354 | * | |
355 | * All cores/threads on this node get this interrupt. | |
356 | * The last one to see it does the s/w ack. | |
357 | * (the resource will not be freed until noninterruptable cpus see this | |
358 | * interrupt; hardware will timeout the s/w ack and reply ERROR) | |
359 | */ | |
b194b120 | 360 | void uv_bau_message_interrupt(struct pt_regs *regs) |
1812924b | 361 | { |
dc163a41 IM |
362 | struct bau_payload_queue_entry *va_queue_first; |
363 | struct bau_payload_queue_entry *va_queue_last; | |
b4c286e6 | 364 | struct bau_payload_queue_entry *msg; |
1812924b | 365 | struct pt_regs *old_regs = set_irq_regs(regs); |
b4c286e6 IM |
366 | cycles_t time1; |
367 | cycles_t time2; | |
1812924b CW |
368 | int msg_slot; |
369 | int sw_ack_slot; | |
370 | int fw; | |
371 | int count = 0; | |
372 | unsigned long local_pnode; | |
373 | ||
374 | ack_APIC_irq(); | |
375 | exit_idle(); | |
376 | irq_enter(); | |
377 | ||
b194b120 | 378 | time1 = get_cycles(); |
1812924b CW |
379 | |
380 | local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); | |
381 | ||
b4c286e6 | 382 | va_queue_first = __get_cpu_var(bau_control).va_queue_first; |
dc163a41 | 383 | va_queue_last = __get_cpu_var(bau_control).va_queue_last; |
b4c286e6 | 384 | |
1812924b CW |
385 | msg = __get_cpu_var(bau_control).bau_msg_head; |
386 | while (msg->sw_ack_vector) { | |
387 | count++; | |
388 | fw = msg->sw_ack_vector; | |
b4c286e6 | 389 | msg_slot = msg - va_queue_first; |
1812924b CW |
390 | sw_ack_slot = ffs(fw) - 1; |
391 | ||
392 | uv_bau_process_message(msg, msg_slot, sw_ack_slot); | |
393 | ||
394 | msg++; | |
dc163a41 IM |
395 | if (msg > va_queue_last) |
396 | msg = va_queue_first; | |
1812924b CW |
397 | __get_cpu_var(bau_control).bau_msg_head = msg; |
398 | } | |
399 | if (!count) | |
400 | __get_cpu_var(ptcstats).nomsg++; | |
401 | else if (count > 1) | |
402 | __get_cpu_var(ptcstats).multmsg++; | |
403 | ||
b194b120 CW |
404 | time2 = get_cycles(); |
405 | __get_cpu_var(ptcstats).dflush += (time2 - time1); | |
1812924b CW |
406 | |
407 | irq_exit(); | |
408 | set_irq_regs(old_regs); | |
1812924b CW |
409 | } |
410 | ||
b194b120 | 411 | static void uv_enable_timeouts(void) |
1812924b CW |
412 | { |
413 | int i; | |
414 | int blade; | |
415 | int last_blade; | |
416 | int pnode; | |
417 | int cur_cpu = 0; | |
418 | unsigned long apicid; | |
419 | ||
1812924b CW |
420 | last_blade = -1; |
421 | for_each_online_node(i) { | |
422 | blade = uv_node_to_blade_id(i); | |
423 | if (blade == last_blade) | |
424 | continue; | |
425 | last_blade = blade; | |
426 | apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); | |
427 | pnode = uv_blade_to_pnode(blade); | |
428 | cur_cpu += uv_blade_nr_possible_cpus(i); | |
429 | } | |
1812924b CW |
430 | } |
431 | ||
b194b120 | 432 | static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset) |
1812924b CW |
433 | { |
434 | if (*offset < num_possible_cpus()) | |
435 | return offset; | |
436 | return NULL; | |
437 | } | |
438 | ||
b194b120 | 439 | static void *uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset) |
1812924b CW |
440 | { |
441 | (*offset)++; | |
442 | if (*offset < num_possible_cpus()) | |
443 | return offset; | |
444 | return NULL; | |
445 | } | |
446 | ||
b194b120 | 447 | static void uv_ptc_seq_stop(struct seq_file *file, void *data) |
1812924b CW |
448 | { |
449 | } | |
450 | ||
451 | /* | |
452 | * Display the statistics thru /proc | |
453 | * data points to the cpu number | |
454 | */ | |
b194b120 | 455 | static int uv_ptc_seq_show(struct seq_file *file, void *data) |
1812924b CW |
456 | { |
457 | struct ptc_stats *stat; | |
458 | int cpu; | |
459 | ||
460 | cpu = *(loff_t *)data; | |
461 | ||
462 | if (!cpu) { | |
463 | seq_printf(file, | |
464 | "# cpu requestor requestee one all sretry dretry ptc_i "); | |
465 | seq_printf(file, | |
b194b120 | 466 | "sw_ack sflush dflush sok dnomsg dmult starget\n"); |
1812924b CW |
467 | } |
468 | if (cpu < num_possible_cpus() && cpu_online(cpu)) { | |
469 | stat = &per_cpu(ptcstats, cpu); | |
470 | seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ", | |
471 | cpu, stat->requestor, | |
472 | stat->requestee, stat->onetlb, stat->alltlb, | |
473 | stat->s_retry, stat->d_retry, stat->ptc_i); | |
474 | seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n", | |
475 | uv_read_global_mmr64(uv_blade_to_pnode | |
476 | (uv_cpu_to_blade_id(cpu)), | |
477 | UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE), | |
b194b120 | 478 | stat->sflush, stat->dflush, |
1812924b CW |
479 | stat->retriesok, stat->nomsg, |
480 | stat->multmsg, stat->ntargeted); | |
481 | } | |
482 | ||
483 | return 0; | |
484 | } | |
485 | ||
486 | /* | |
487 | * 0: display meaning of the statistics | |
488 | * >0: retry limit | |
489 | */ | |
b194b120 | 490 | static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, |
b4c286e6 | 491 | size_t count, loff_t *data) |
1812924b CW |
492 | { |
493 | long newmode; | |
494 | char optstr[64]; | |
495 | ||
e7eb8726 | 496 | if (count == 0 || count > sizeof(optstr)) |
cef53278 | 497 | return -EINVAL; |
1812924b CW |
498 | if (copy_from_user(optstr, user, count)) |
499 | return -EFAULT; | |
500 | optstr[count - 1] = '\0'; | |
501 | if (strict_strtoul(optstr, 10, &newmode) < 0) { | |
502 | printk(KERN_DEBUG "%s is invalid\n", optstr); | |
503 | return -EINVAL; | |
504 | } | |
505 | ||
506 | if (newmode == 0) { | |
507 | printk(KERN_DEBUG "# cpu: cpu number\n"); | |
508 | printk(KERN_DEBUG | |
509 | "requestor: times this cpu was the flush requestor\n"); | |
510 | printk(KERN_DEBUG | |
511 | "requestee: times this cpu was requested to flush its TLBs\n"); | |
512 | printk(KERN_DEBUG | |
513 | "one: times requested to flush a single address\n"); | |
514 | printk(KERN_DEBUG | |
515 | "all: times requested to flush all TLB's\n"); | |
516 | printk(KERN_DEBUG | |
517 | "sretry: number of retries of source-side timeouts\n"); | |
518 | printk(KERN_DEBUG | |
519 | "dretry: number of retries of destination-side timeouts\n"); | |
520 | printk(KERN_DEBUG | |
521 | "ptc_i: times UV fell through to IPI-style flushes\n"); | |
522 | printk(KERN_DEBUG | |
523 | "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n"); | |
524 | printk(KERN_DEBUG | |
b194b120 | 525 | "sflush_us: cycles spent in uv_flush_tlb_others()\n"); |
1812924b | 526 | printk(KERN_DEBUG |
b194b120 | 527 | "dflush_us: cycles spent in handling flush requests\n"); |
1812924b CW |
528 | printk(KERN_DEBUG "sok: successes on retry\n"); |
529 | printk(KERN_DEBUG "dnomsg: interrupts with no message\n"); | |
530 | printk(KERN_DEBUG | |
531 | "dmult: interrupts with multiple messages\n"); | |
532 | printk(KERN_DEBUG "starget: nodes targeted\n"); | |
533 | } else { | |
534 | uv_bau_retry_limit = newmode; | |
535 | printk(KERN_DEBUG "timeout retry limit:%d\n", | |
536 | uv_bau_retry_limit); | |
537 | } | |
538 | ||
539 | return count; | |
540 | } | |
541 | ||
542 | static const struct seq_operations uv_ptc_seq_ops = { | |
dc163a41 IM |
543 | .start = uv_ptc_seq_start, |
544 | .next = uv_ptc_seq_next, | |
545 | .stop = uv_ptc_seq_stop, | |
546 | .show = uv_ptc_seq_show | |
1812924b CW |
547 | }; |
548 | ||
b194b120 | 549 | static int uv_ptc_proc_open(struct inode *inode, struct file *file) |
1812924b CW |
550 | { |
551 | return seq_open(file, &uv_ptc_seq_ops); | |
552 | } | |
553 | ||
554 | static const struct file_operations proc_uv_ptc_operations = { | |
b194b120 CW |
555 | .open = uv_ptc_proc_open, |
556 | .read = seq_read, | |
557 | .write = uv_ptc_proc_write, | |
558 | .llseek = seq_lseek, | |
559 | .release = seq_release, | |
1812924b CW |
560 | }; |
561 | ||
b194b120 | 562 | static int __init uv_ptc_init(void) |
1812924b | 563 | { |
b194b120 | 564 | struct proc_dir_entry *proc_uv_ptc; |
1812924b CW |
565 | |
566 | if (!is_uv_system()) | |
567 | return 0; | |
568 | ||
b194b120 | 569 | if (!proc_mkdir("sgi_uv", NULL)) |
1812924b CW |
570 | return -EINVAL; |
571 | ||
572 | proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL); | |
573 | if (!proc_uv_ptc) { | |
574 | printk(KERN_ERR "unable to create %s proc entry\n", | |
575 | UV_PTC_BASENAME); | |
dc163a41 | 576 | remove_proc_entry("sgi_uv", NULL); |
1812924b CW |
577 | return -EINVAL; |
578 | } | |
579 | proc_uv_ptc->proc_fops = &proc_uv_ptc_operations; | |
580 | return 0; | |
581 | } | |
582 | ||
b194b120 CW |
583 | /* |
584 | * begin the initialization of the per-blade control structures | |
585 | */ | |
586 | static struct bau_control * __init uv_table_bases_init(int blade, int node) | |
1812924b | 587 | { |
b194b120 CW |
588 | int i; |
589 | int *ip; | |
590 | struct bau_msg_status *msp; | |
dc163a41 | 591 | struct bau_control *bau_tabp; |
b194b120 | 592 | |
dc163a41 | 593 | bau_tabp = |
b194b120 | 594 | kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node); |
dc163a41 | 595 | BUG_ON(!bau_tabp); |
b4c286e6 | 596 | |
dc163a41 | 597 | bau_tabp->msg_statuses = |
b194b120 | 598 | kmalloc_node(sizeof(struct bau_msg_status) * |
dc163a41 IM |
599 | DEST_Q_SIZE, GFP_KERNEL, node); |
600 | BUG_ON(!bau_tabp->msg_statuses); | |
b4c286e6 | 601 | |
dc163a41 | 602 | for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++) |
b194b120 CW |
603 | bau_cpubits_clear(&msp->seen_by, (int) |
604 | uv_blade_nr_possible_cpus(blade)); | |
b4c286e6 | 605 | |
dc163a41 IM |
606 | bau_tabp->watching = |
607 | kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node); | |
608 | BUG_ON(!bau_tabp->watching); | |
b4c286e6 IM |
609 | |
610 | for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++) | |
b194b120 | 611 | *ip = 0; |
b4c286e6 | 612 | |
dc163a41 | 613 | uv_bau_table_bases[blade] = bau_tabp; |
b4c286e6 | 614 | |
d400524a | 615 | return bau_tabp; |
1812924b CW |
616 | } |
617 | ||
b194b120 CW |
618 | /* |
619 | * finish the initialization of the per-blade control structures | |
620 | */ | |
b4c286e6 IM |
621 | static void __init |
622 | uv_table_bases_finish(int blade, int node, int cur_cpu, | |
623 | struct bau_control *bau_tablesp, | |
624 | struct bau_desc *adp) | |
b194b120 | 625 | { |
b194b120 | 626 | struct bau_control *bcp; |
b4c286e6 | 627 | int i; |
b194b120 | 628 | |
b4c286e6 | 629 | for (i = cur_cpu; i < cur_cpu + uv_blade_nr_possible_cpus(blade); i++) { |
b194b120 | 630 | bcp = (struct bau_control *)&per_cpu(bau_control, i); |
b4c286e6 IM |
631 | |
632 | bcp->bau_msg_head = bau_tablesp->va_queue_first; | |
633 | bcp->va_queue_first = bau_tablesp->va_queue_first; | |
634 | bcp->va_queue_last = bau_tablesp->va_queue_last; | |
635 | bcp->watching = bau_tablesp->watching; | |
636 | bcp->msg_statuses = bau_tablesp->msg_statuses; | |
637 | bcp->descriptor_base = adp; | |
b194b120 CW |
638 | } |
639 | } | |
1812924b CW |
640 | |
641 | /* | |
b194b120 | 642 | * initialize the sending side's sending buffers |
1812924b | 643 | */ |
dc163a41 | 644 | static struct bau_desc * __init |
b194b120 | 645 | uv_activation_descriptor_init(int node, int pnode) |
1812924b CW |
646 | { |
647 | int i; | |
1812924b | 648 | unsigned long pa; |
1812924b | 649 | unsigned long m; |
b194b120 | 650 | unsigned long n; |
1812924b | 651 | unsigned long mmr_image; |
dc163a41 IM |
652 | struct bau_desc *adp; |
653 | struct bau_desc *ad2; | |
b194b120 | 654 | |
dc163a41 | 655 | adp = (struct bau_desc *) |
b194b120 | 656 | kmalloc_node(16384, GFP_KERNEL, node); |
dc163a41 | 657 | BUG_ON(!adp); |
b4c286e6 | 658 | |
b194b120 CW |
659 | pa = __pa((unsigned long)adp); |
660 | n = pa >> uv_nshift; | |
661 | m = pa & uv_mmask; | |
b4c286e6 | 662 | |
b194b120 | 663 | mmr_image = uv_read_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE); |
b4c286e6 | 664 | if (mmr_image) { |
b194b120 CW |
665 | uv_write_global_mmr64(pnode, (unsigned long) |
666 | UVH_LB_BAU_SB_DESCRIPTOR_BASE, | |
667 | (n << UV_DESC_BASE_PNODE_SHIFT | m)); | |
b4c286e6 IM |
668 | } |
669 | ||
b194b120 | 670 | for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) { |
dc163a41 | 671 | memset(ad2, 0, sizeof(struct bau_desc)); |
b194b120 CW |
672 | ad2->header.sw_ack_flag = 1; |
673 | ad2->header.base_dest_nodeid = | |
674 | uv_blade_to_pnode(uv_cpu_to_blade_id(0)); | |
675 | ad2->header.command = UV_NET_ENDPOINT_INTD; | |
676 | ad2->header.int_both = 1; | |
677 | /* | |
678 | * all others need to be set to zero: | |
679 | * fairness chaining multilevel count replied_to | |
680 | */ | |
681 | } | |
682 | return adp; | |
683 | } | |
684 | ||
685 | /* | |
686 | * initialize the destination side's receiving buffers | |
687 | */ | |
b4c286e6 IM |
688 | static struct bau_payload_queue_entry * __init |
689 | uv_payload_queue_init(int node, int pnode, struct bau_control *bau_tablesp) | |
b194b120 | 690 | { |
1812924b | 691 | struct bau_payload_queue_entry *pqp; |
b4c286e6 | 692 | char *cp; |
1812924b | 693 | |
dc163a41 IM |
694 | pqp = (struct bau_payload_queue_entry *) kmalloc_node( |
695 | (DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry), | |
696 | GFP_KERNEL, node); | |
697 | BUG_ON(!pqp); | |
b4c286e6 | 698 | |
b194b120 CW |
699 | cp = (char *)pqp + 31; |
700 | pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5); | |
701 | bau_tablesp->va_queue_first = pqp; | |
702 | uv_write_global_mmr64(pnode, | |
703 | UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, | |
704 | ((unsigned long)pnode << | |
705 | UV_PAYLOADQ_PNODE_SHIFT) | | |
706 | uv_physnodeaddr(pqp)); | |
707 | uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, | |
708 | uv_physnodeaddr(pqp)); | |
dc163a41 | 709 | bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1); |
b194b120 CW |
710 | uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, |
711 | (unsigned long) | |
712 | uv_physnodeaddr(bau_tablesp->va_queue_last)); | |
dc163a41 | 713 | memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE); |
b4c286e6 | 714 | |
b194b120 CW |
715 | return pqp; |
716 | } | |
1812924b | 717 | |
b194b120 CW |
718 | /* |
719 | * Initialization of each UV blade's structures | |
720 | */ | |
721 | static int __init uv_init_blade(int blade, int node, int cur_cpu) | |
722 | { | |
723 | int pnode; | |
724 | unsigned long pa; | |
725 | unsigned long apicid; | |
dc163a41 | 726 | struct bau_desc *adp; |
b194b120 CW |
727 | struct bau_payload_queue_entry *pqp; |
728 | struct bau_control *bau_tablesp; | |
1812924b | 729 | |
b194b120 CW |
730 | bau_tablesp = uv_table_bases_init(blade, node); |
731 | pnode = uv_blade_to_pnode(blade); | |
732 | adp = uv_activation_descriptor_init(node, pnode); | |
733 | pqp = uv_payload_queue_init(node, pnode, bau_tablesp); | |
734 | uv_table_bases_finish(blade, node, cur_cpu, bau_tablesp, adp); | |
735 | /* | |
736 | * the below initialization can't be in firmware because the | |
737 | * messaging IRQ will be determined by the OS | |
738 | */ | |
739 | apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); | |
740 | pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG); | |
741 | if ((pa & 0xff) != UV_BAU_MESSAGE) { | |
742 | uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, | |
743 | ((apicid << 32) | UV_BAU_MESSAGE)); | |
1812924b | 744 | } |
b194b120 CW |
745 | return 0; |
746 | } | |
747 | ||
748 | /* | |
749 | * Initialization of BAU-related structures | |
750 | */ | |
751 | static int __init uv_bau_init(void) | |
752 | { | |
753 | int blade; | |
754 | int node; | |
755 | int nblades; | |
756 | int last_blade; | |
757 | int cur_cpu = 0; | |
758 | ||
759 | if (!is_uv_system()) | |
760 | return 0; | |
1812924b | 761 | |
b194b120 | 762 | uv_bau_retry_limit = 1; |
1812924b | 763 | uv_nshift = uv_hub_info->n_val; |
dc163a41 | 764 | uv_mmask = (1UL << uv_hub_info->n_val) - 1; |
1812924b CW |
765 | nblades = 0; |
766 | last_blade = -1; | |
b194b120 CW |
767 | for_each_online_node(node) { |
768 | blade = uv_node_to_blade_id(node); | |
1812924b CW |
769 | if (blade == last_blade) |
770 | continue; | |
771 | last_blade = blade; | |
772 | nblades++; | |
773 | } | |
1812924b CW |
774 | uv_bau_table_bases = (struct bau_control **) |
775 | kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); | |
dc163a41 | 776 | BUG_ON(!uv_bau_table_bases); |
b4c286e6 | 777 | |
1812924b | 778 | last_blade = -1; |
b194b120 CW |
779 | for_each_online_node(node) { |
780 | blade = uv_node_to_blade_id(node); | |
1812924b CW |
781 | if (blade == last_blade) |
782 | continue; | |
783 | last_blade = blade; | |
b194b120 CW |
784 | uv_init_blade(blade, node, cur_cpu); |
785 | cur_cpu += uv_blade_nr_possible_cpus(blade); | |
1812924b | 786 | } |
99dd8713 | 787 | alloc_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1); |
1812924b | 788 | uv_enable_timeouts(); |
b4c286e6 | 789 | |
1812924b CW |
790 | return 0; |
791 | } | |
1812924b | 792 | __initcall(uv_bau_init); |
b194b120 | 793 | __initcall(uv_ptc_init); |