Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* $Id: ultra.S,v 1.72 2002/02/09 19:49:31 davem Exp $ |
2 | * ultra.S: Don't expand these all over the place... | |
3 | * | |
4 | * Copyright (C) 1997, 2000 David S. Miller (davem@redhat.com) | |
5 | */ | |
6 | ||
7 | #include <linux/config.h> | |
8 | #include <asm/asi.h> | |
9 | #include <asm/pgtable.h> | |
10 | #include <asm/page.h> | |
11 | #include <asm/spitfire.h> | |
12 | #include <asm/mmu_context.h> | |
2ef27778 | 13 | #include <asm/mmu.h> |
1da177e4 LT |
14 | #include <asm/pil.h> |
15 | #include <asm/head.h> | |
16 | #include <asm/thread_info.h> | |
17 | #include <asm/cacheflush.h> | |
18 | ||
19 | /* Basically, most of the Spitfire vs. Cheetah madness | |
20 | * has to do with the fact that Cheetah does not support | |
21 | * IMMU flushes out of the secondary context. Someone needs | |
22 | * to throw a south lake birthday party for the folks | |
23 | * in Microelectronics who refused to fix this shit. | |
24 | */ | |
25 | ||
26 | /* This file is meant to be read efficiently by the CPU, not humans. | |
27 | * Staraj sie tego nikomu nie pierdolnac... | |
28 | */ | |
29 | .text | |
30 | .align 32 | |
31 | .globl __flush_tlb_mm | |
32 | __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ | |
33 | ldxa [%o1] ASI_DMMU, %g2 | |
34 | cmp %g2, %o0 | |
35 | bne,pn %icc, __spitfire_flush_tlb_mm_slow | |
36 | mov 0x50, %g3 | |
37 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
38 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
39 | retl | |
40 | flush %g6 | |
41 | nop | |
42 | nop | |
43 | nop | |
44 | nop | |
45 | nop | |
46 | nop | |
47 | nop | |
48 | nop | |
2ef27778 DM |
49 | nop |
50 | nop | |
1da177e4 LT |
51 | |
52 | .align 32 | |
53 | .globl __flush_tlb_pending | |
54 | __flush_tlb_pending: | |
55 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ | |
56 | rdpr %pstate, %g7 | |
57 | sllx %o1, 3, %o1 | |
58 | andn %g7, PSTATE_IE, %g2 | |
59 | wrpr %g2, %pstate | |
60 | mov SECONDARY_CONTEXT, %o4 | |
61 | ldxa [%o4] ASI_DMMU, %g2 | |
62 | stxa %o0, [%o4] ASI_DMMU | |
63 | 1: sub %o1, (1 << 3), %o1 | |
64 | ldx [%o2 + %o1], %o3 | |
65 | andcc %o3, 1, %g0 | |
66 | andn %o3, 1, %o3 | |
67 | be,pn %icc, 2f | |
68 | or %o3, 0x10, %o3 | |
69 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
70 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
71 | membar #Sync | |
72 | brnz,pt %o1, 1b | |
73 | nop | |
74 | stxa %g2, [%o4] ASI_DMMU | |
75 | flush %g6 | |
76 | retl | |
77 | wrpr %g7, 0x0, %pstate | |
fef43da4 | 78 | nop |
2ef27778 DM |
79 | nop |
80 | nop | |
81 | nop | |
1da177e4 LT |
82 | |
83 | .align 32 | |
84 | .globl __flush_tlb_kernel_range | |
85 | __flush_tlb_kernel_range: /* %o0=start, %o1=end */ | |
86 | cmp %o0, %o1 | |
87 | be,pn %xcc, 2f | |
88 | sethi %hi(PAGE_SIZE), %o4 | |
89 | sub %o1, %o0, %o3 | |
90 | sub %o3, %o4, %o3 | |
91 | or %o0, 0x20, %o0 ! Nucleus | |
92 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP | |
93 | stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP | |
94 | membar #Sync | |
95 | brnz,pt %o3, 1b | |
96 | sub %o3, %o4, %o3 | |
97 | 2: retl | |
98 | flush %g6 | |
99 | ||
100 | __spitfire_flush_tlb_mm_slow: | |
101 | rdpr %pstate, %g1 | |
102 | wrpr %g1, PSTATE_IE, %pstate | |
103 | stxa %o0, [%o1] ASI_DMMU | |
104 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
105 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
106 | flush %g6 | |
107 | stxa %g2, [%o1] ASI_DMMU | |
108 | flush %g6 | |
109 | retl | |
110 | wrpr %g1, 0, %pstate | |
111 | ||
112 | /* | |
113 | * The following code flushes one page_size worth. | |
114 | */ | |
115 | #if (PAGE_SHIFT == 13) | |
116 | #define ITAG_MASK 0xfe | |
117 | #elif (PAGE_SHIFT == 16) | |
118 | #define ITAG_MASK 0x7fe | |
119 | #else | |
120 | #error unsupported PAGE_SIZE | |
121 | #endif | |
83005161 | 122 | .section .kprobes.text, "ax" |
1da177e4 LT |
123 | .align 32 |
124 | .globl __flush_icache_page | |
125 | __flush_icache_page: /* %o0 = phys_page */ | |
126 | membar #StoreStore | |
127 | srlx %o0, PAGE_SHIFT, %o0 | |
128 | sethi %uhi(PAGE_OFFSET), %g1 | |
129 | sllx %o0, PAGE_SHIFT, %o0 | |
130 | sethi %hi(PAGE_SIZE), %g2 | |
131 | sllx %g1, 32, %g1 | |
132 | add %o0, %g1, %o0 | |
133 | 1: subcc %g2, 32, %g2 | |
134 | bne,pt %icc, 1b | |
135 | flush %o0 + %g2 | |
136 | retl | |
137 | nop | |
138 | ||
139 | #ifdef DCACHE_ALIASING_POSSIBLE | |
140 | ||
141 | #if (PAGE_SHIFT != 13) | |
142 | #error only page shift of 13 is supported by dcache flush | |
143 | #endif | |
144 | ||
145 | #define DTAG_MASK 0x3 | |
146 | ||
c5bd50a9 DM |
147 | /* This routine is Spitfire specific so the hardcoded |
148 | * D-cache size and line-size are OK. | |
149 | */ | |
1da177e4 LT |
150 | .align 64 |
151 | .globl __flush_dcache_page | |
152 | __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ | |
153 | sethi %uhi(PAGE_OFFSET), %g1 | |
154 | sllx %g1, 32, %g1 | |
c5bd50a9 DM |
155 | sub %o0, %g1, %o0 ! physical address |
156 | srlx %o0, 11, %o0 ! make D-cache TAG | |
157 | sethi %hi(1 << 14), %o2 ! D-cache size | |
158 | sub %o2, (1 << 5), %o2 ! D-cache line size | |
159 | 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG | |
160 | andcc %o3, DTAG_MASK, %g0 ! Valid? | |
161 | be,pn %xcc, 2f ! Nope, branch | |
162 | andn %o3, DTAG_MASK, %o3 ! Clear valid bits | |
163 | cmp %o3, %o0 ! TAG match? | |
164 | bne,pt %xcc, 2f ! Nope, branch | |
165 | nop | |
166 | stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG | |
167 | membar #Sync | |
168 | 2: brnz,pt %o2, 1b | |
169 | sub %o2, (1 << 5), %o2 ! D-cache line size | |
1da177e4 LT |
170 | |
171 | /* The I-cache does not snoop local stores so we | |
172 | * better flush that too when necessary. | |
173 | */ | |
174 | brnz,pt %o1, __flush_icache_page | |
175 | sllx %o0, 11, %o0 | |
176 | retl | |
177 | nop | |
178 | ||
1da177e4 LT |
179 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
180 | ||
c5bd50a9 DM |
181 | .previous |
182 | ||
2ef27778 DM |
183 | /* Cheetah specific versions, patched at boot time. */ |
184 | __cheetah_flush_tlb_mm: /* 18 insns */ | |
1da177e4 LT |
185 | rdpr %pstate, %g7 |
186 | andn %g7, PSTATE_IE, %g2 | |
187 | wrpr %g2, 0x0, %pstate | |
188 | wrpr %g0, 1, %tl | |
189 | mov PRIMARY_CONTEXT, %o2 | |
190 | mov 0x40, %g3 | |
191 | ldxa [%o2] ASI_DMMU, %g2 | |
2ef27778 DM |
192 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 |
193 | sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 | |
194 | or %o0, %o1, %o0 /* Preserve nucleus page size fields */ | |
1da177e4 LT |
195 | stxa %o0, [%o2] ASI_DMMU |
196 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
197 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
198 | stxa %g2, [%o2] ASI_DMMU | |
199 | flush %g6 | |
200 | wrpr %g0, 0, %tl | |
201 | retl | |
202 | wrpr %g7, 0x0, %pstate | |
203 | ||
2ef27778 | 204 | __cheetah_flush_tlb_pending: /* 26 insns */ |
1da177e4 LT |
205 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
206 | rdpr %pstate, %g7 | |
207 | sllx %o1, 3, %o1 | |
208 | andn %g7, PSTATE_IE, %g2 | |
209 | wrpr %g2, 0x0, %pstate | |
210 | wrpr %g0, 1, %tl | |
211 | mov PRIMARY_CONTEXT, %o4 | |
212 | ldxa [%o4] ASI_DMMU, %g2 | |
2ef27778 DM |
213 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 |
214 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 | |
215 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ | |
1da177e4 LT |
216 | stxa %o0, [%o4] ASI_DMMU |
217 | 1: sub %o1, (1 << 3), %o1 | |
218 | ldx [%o2 + %o1], %o3 | |
219 | andcc %o3, 1, %g0 | |
220 | be,pn %icc, 2f | |
221 | andn %o3, 1, %o3 | |
222 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
223 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
b445e26c | 224 | membar #Sync |
1da177e4 | 225 | brnz,pt %o1, 1b |
b445e26c | 226 | nop |
1da177e4 LT |
227 | stxa %g2, [%o4] ASI_DMMU |
228 | flush %g6 | |
229 | wrpr %g0, 0, %tl | |
230 | retl | |
231 | wrpr %g7, 0x0, %pstate | |
232 | ||
233 | #ifdef DCACHE_ALIASING_POSSIBLE | |
c5bd50a9 | 234 | __cheetah_flush_dcache_page: /* 11 insns */ |
1da177e4 LT |
235 | sethi %uhi(PAGE_OFFSET), %g1 |
236 | sllx %g1, 32, %g1 | |
237 | sub %o0, %g1, %o0 | |
238 | sethi %hi(PAGE_SIZE), %o4 | |
239 | 1: subcc %o4, (1 << 5), %o4 | |
240 | stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE | |
241 | membar #Sync | |
242 | bne,pt %icc, 1b | |
243 | nop | |
244 | retl /* I-cache flush never needed on Cheetah, see callers. */ | |
245 | nop | |
246 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
247 | ||
248 | cheetah_patch_one: | |
249 | 1: lduw [%o1], %g1 | |
250 | stw %g1, [%o0] | |
251 | flush %o0 | |
252 | subcc %o2, 1, %o2 | |
253 | add %o1, 4, %o1 | |
254 | bne,pt %icc, 1b | |
255 | add %o0, 4, %o0 | |
256 | retl | |
257 | nop | |
258 | ||
259 | .globl cheetah_patch_cachetlbops | |
260 | cheetah_patch_cachetlbops: | |
261 | save %sp, -128, %sp | |
262 | ||
263 | sethi %hi(__flush_tlb_mm), %o0 | |
264 | or %o0, %lo(__flush_tlb_mm), %o0 | |
265 | sethi %hi(__cheetah_flush_tlb_mm), %o1 | |
266 | or %o1, %lo(__cheetah_flush_tlb_mm), %o1 | |
267 | call cheetah_patch_one | |
2ef27778 | 268 | mov 18, %o2 |
1da177e4 LT |
269 | |
270 | sethi %hi(__flush_tlb_pending), %o0 | |
271 | or %o0, %lo(__flush_tlb_pending), %o0 | |
272 | sethi %hi(__cheetah_flush_tlb_pending), %o1 | |
273 | or %o1, %lo(__cheetah_flush_tlb_pending), %o1 | |
274 | call cheetah_patch_one | |
2ef27778 | 275 | mov 26, %o2 |
1da177e4 LT |
276 | |
277 | #ifdef DCACHE_ALIASING_POSSIBLE | |
278 | sethi %hi(__flush_dcache_page), %o0 | |
279 | or %o0, %lo(__flush_dcache_page), %o0 | |
c5bd50a9 DM |
280 | sethi %hi(__cheetah_flush_dcache_page), %o1 |
281 | or %o1, %lo(__cheetah_flush_dcache_page), %o1 | |
1da177e4 LT |
282 | call cheetah_patch_one |
283 | mov 11, %o2 | |
284 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
285 | ||
286 | ret | |
287 | restore | |
288 | ||
289 | #ifdef CONFIG_SMP | |
290 | /* These are all called by the slaves of a cross call, at | |
291 | * trap level 1, with interrupts fully disabled. | |
292 | * | |
293 | * Register usage: | |
294 | * %g5 mm->context (all tlb flushes) | |
295 | * %g1 address arg 1 (tlb page and range flushes) | |
296 | * %g7 address arg 2 (tlb range flush only) | |
297 | * | |
298 | * %g6 ivector table, don't touch | |
299 | * %g2 scratch 1 | |
300 | * %g3 scratch 2 | |
301 | * %g4 scratch 3 | |
302 | * | |
303 | * TODO: Make xcall TLB range flushes use the tricks above... -DaveM | |
304 | */ | |
305 | .align 32 | |
306 | .globl xcall_flush_tlb_mm | |
307 | xcall_flush_tlb_mm: | |
308 | mov PRIMARY_CONTEXT, %g2 | |
1da177e4 | 309 | ldxa [%g2] ASI_DMMU, %g3 |
2ef27778 DM |
310 | srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 |
311 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 | |
312 | or %g5, %g4, %g5 /* Preserve nucleus page size fields */ | |
1da177e4 | 313 | stxa %g5, [%g2] ASI_DMMU |
2ef27778 | 314 | mov 0x40, %g4 |
1da177e4 LT |
315 | stxa %g0, [%g4] ASI_DMMU_DEMAP |
316 | stxa %g0, [%g4] ASI_IMMU_DEMAP | |
317 | stxa %g3, [%g2] ASI_DMMU | |
318 | retry | |
319 | ||
320 | .globl xcall_flush_tlb_pending | |
321 | xcall_flush_tlb_pending: | |
322 | /* %g5=context, %g1=nr, %g7=vaddrs[] */ | |
323 | sllx %g1, 3, %g1 | |
324 | mov PRIMARY_CONTEXT, %g4 | |
325 | ldxa [%g4] ASI_DMMU, %g2 | |
2ef27778 DM |
326 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 |
327 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 | |
328 | or %g5, %g4, %g5 | |
329 | mov PRIMARY_CONTEXT, %g4 | |
1da177e4 LT |
330 | stxa %g5, [%g4] ASI_DMMU |
331 | 1: sub %g1, (1 << 3), %g1 | |
332 | ldx [%g7 + %g1], %g5 | |
333 | andcc %g5, 0x1, %g0 | |
334 | be,pn %icc, 2f | |
335 | ||
336 | andn %g5, 0x1, %g5 | |
337 | stxa %g0, [%g5] ASI_IMMU_DEMAP | |
338 | 2: stxa %g0, [%g5] ASI_DMMU_DEMAP | |
339 | membar #Sync | |
340 | brnz,pt %g1, 1b | |
341 | nop | |
342 | stxa %g2, [%g4] ASI_DMMU | |
343 | retry | |
344 | ||
345 | .globl xcall_flush_tlb_kernel_range | |
346 | xcall_flush_tlb_kernel_range: | |
347 | sethi %hi(PAGE_SIZE - 1), %g2 | |
348 | or %g2, %lo(PAGE_SIZE - 1), %g2 | |
349 | andn %g1, %g2, %g1 | |
350 | andn %g7, %g2, %g7 | |
351 | sub %g7, %g1, %g3 | |
352 | add %g2, 1, %g2 | |
353 | sub %g3, %g2, %g3 | |
354 | or %g1, 0x20, %g1 ! Nucleus | |
355 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP | |
356 | stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP | |
357 | membar #Sync | |
358 | brnz,pt %g3, 1b | |
359 | sub %g3, %g2, %g3 | |
360 | retry | |
361 | nop | |
362 | nop | |
363 | ||
364 | /* This runs in a very controlled environment, so we do | |
365 | * not need to worry about BH races etc. | |
366 | */ | |
367 | .globl xcall_sync_tick | |
368 | xcall_sync_tick: | |
369 | rdpr %pstate, %g2 | |
370 | wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate | |
371 | rdpr %pil, %g2 | |
372 | wrpr %g0, 15, %pil | |
373 | sethi %hi(109f), %g7 | |
374 | b,pt %xcc, etrap_irq | |
375 | 109: or %g7, %lo(109b), %g7 | |
376 | call smp_synchronize_tick_client | |
377 | nop | |
378 | clr %l6 | |
379 | b rtrap_xcall | |
380 | ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 | |
381 | ||
382 | /* NOTE: This is SPECIAL!! We do etrap/rtrap however | |
383 | * we choose to deal with the "BH's run with | |
384 | * %pil==15" problem (described in asm/pil.h) | |
385 | * by just invoking rtrap directly past where | |
386 | * BH's are checked for. | |
387 | * | |
388 | * We do it like this because we do not want %pil==15 | |
389 | * lockups to prevent regs being reported. | |
390 | */ | |
391 | .globl xcall_report_regs | |
392 | xcall_report_regs: | |
393 | rdpr %pstate, %g2 | |
394 | wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate | |
395 | rdpr %pil, %g2 | |
396 | wrpr %g0, 15, %pil | |
397 | sethi %hi(109f), %g7 | |
398 | b,pt %xcc, etrap_irq | |
399 | 109: or %g7, %lo(109b), %g7 | |
400 | call __show_regs | |
401 | add %sp, PTREGS_OFF, %o0 | |
402 | clr %l6 | |
403 | /* Has to be a non-v9 branch due to the large distance. */ | |
404 | b rtrap_xcall | |
405 | ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 | |
406 | ||
407 | #ifdef DCACHE_ALIASING_POSSIBLE | |
408 | .align 32 | |
409 | .globl xcall_flush_dcache_page_cheetah | |
410 | xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ | |
411 | sethi %hi(PAGE_SIZE), %g3 | |
412 | 1: subcc %g3, (1 << 5), %g3 | |
413 | stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE | |
414 | membar #Sync | |
415 | bne,pt %icc, 1b | |
416 | nop | |
417 | retry | |
418 | nop | |
419 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
420 | ||
421 | .globl xcall_flush_dcache_page_spitfire | |
422 | xcall_flush_dcache_page_spitfire: /* %g1 == physical page address | |
423 | %g7 == kernel page virtual address | |
424 | %g5 == (page->mapping != NULL) */ | |
425 | #ifdef DCACHE_ALIASING_POSSIBLE | |
426 | srlx %g1, (13 - 2), %g1 ! Form tag comparitor | |
427 | sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K | |
428 | sub %g3, (1 << 5), %g3 ! D$ linesize == 32 | |
429 | 1: ldxa [%g3] ASI_DCACHE_TAG, %g2 | |
430 | andcc %g2, 0x3, %g0 | |
431 | be,pn %xcc, 2f | |
432 | andn %g2, 0x3, %g2 | |
433 | cmp %g2, %g1 | |
434 | ||
435 | bne,pt %xcc, 2f | |
436 | nop | |
437 | stxa %g0, [%g3] ASI_DCACHE_TAG | |
438 | membar #Sync | |
439 | 2: cmp %g3, 0 | |
440 | bne,pt %xcc, 1b | |
441 | sub %g3, (1 << 5), %g3 | |
442 | ||
443 | brz,pn %g5, 2f | |
444 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
445 | sethi %hi(PAGE_SIZE), %g3 | |
446 | ||
447 | 1: flush %g7 | |
448 | subcc %g3, (1 << 5), %g3 | |
449 | bne,pt %icc, 1b | |
450 | add %g7, (1 << 5), %g7 | |
451 | ||
452 | 2: retry | |
453 | nop | |
454 | nop | |
455 | ||
1da177e4 LT |
456 | /* These just get rescheduled to PIL vectors. */ |
457 | .globl xcall_call_function | |
458 | xcall_call_function: | |
459 | wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint | |
460 | retry | |
461 | ||
462 | .globl xcall_receive_signal | |
463 | xcall_receive_signal: | |
464 | wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint | |
465 | retry | |
466 | ||
467 | .globl xcall_capture | |
468 | xcall_capture: | |
469 | wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint | |
470 | retry | |
471 | ||
472 | #endif /* CONFIG_SMP */ |