Commit | Line | Data |
---|---|---|
b00dc837 | 1 | /* |
1da177e4 LT |
2 | * ultra.S: Don't expand these all over the place... |
3 | * | |
93dae5b7 | 4 | * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) |
1da177e4 LT |
5 | */ |
6 | ||
1da177e4 LT |
7 | #include <asm/asi.h> |
8 | #include <asm/pgtable.h> | |
9 | #include <asm/page.h> | |
10 | #include <asm/spitfire.h> | |
11 | #include <asm/mmu_context.h> | |
2ef27778 | 12 | #include <asm/mmu.h> |
1da177e4 LT |
13 | #include <asm/pil.h> |
14 | #include <asm/head.h> | |
15 | #include <asm/thread_info.h> | |
16 | #include <asm/cacheflush.h> | |
52bf082f | 17 | #include <asm/hypervisor.h> |
93dae5b7 | 18 | #include <asm/cpudata.h> |
1da177e4 LT |
19 | |
20 | /* Basically, most of the Spitfire vs. Cheetah madness | |
21 | * has to do with the fact that Cheetah does not support | |
22 | * IMMU flushes out of the secondary context. Someone needs | |
23 | * to throw a south lake birthday party for the folks | |
24 | * in Microelectronics who refused to fix this shit. | |
25 | */ | |
26 | ||
27 | /* This file is meant to be read efficiently by the CPU, not humans. | |
28 | * Staraj sie tego nikomu nie pierdolnac... | |
29 | */ | |
30 | .text | |
31 | .align 32 | |
32 | .globl __flush_tlb_mm | |
52bf082f DM |
33 | __flush_tlb_mm: /* 18 insns */ |
34 | /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ | |
1da177e4 LT |
35 | ldxa [%o1] ASI_DMMU, %g2 |
36 | cmp %g2, %o0 | |
37 | bne,pn %icc, __spitfire_flush_tlb_mm_slow | |
38 | mov 0x50, %g3 | |
39 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
40 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
4da808c3 DM |
41 | sethi %hi(KERNBASE), %g3 |
42 | flush %g3 | |
1da177e4 | 43 | retl |
4da808c3 | 44 | nop |
1da177e4 LT |
45 | nop |
46 | nop | |
47 | nop | |
48 | nop | |
49 | nop | |
50 | nop | |
51 | nop | |
2ef27778 DM |
52 | nop |
53 | nop | |
1da177e4 | 54 | |
f36391d2 DM |
55 | .align 32 |
56 | .globl __flush_tlb_page | |
57 | __flush_tlb_page: /* 22 insns */ | |
58 | /* %o0 = context, %o1 = vaddr */ | |
59 | rdpr %pstate, %g7 | |
60 | andn %g7, PSTATE_IE, %g2 | |
61 | wrpr %g2, %pstate | |
62 | mov SECONDARY_CONTEXT, %o4 | |
63 | ldxa [%o4] ASI_DMMU, %g2 | |
64 | stxa %o0, [%o4] ASI_DMMU | |
65 | andcc %o1, 1, %g0 | |
66 | andn %o1, 1, %o3 | |
67 | be,pn %icc, 1f | |
68 | or %o3, 0x10, %o3 | |
69 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
70 | 1: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
71 | membar #Sync | |
72 | stxa %g2, [%o4] ASI_DMMU | |
73 | sethi %hi(KERNBASE), %o4 | |
74 | flush %o4 | |
75 | retl | |
76 | wrpr %g7, 0x0, %pstate | |
77 | nop | |
78 | nop | |
79 | nop | |
80 | nop | |
81 | ||
1da177e4 LT |
82 | .align 32 |
83 | .globl __flush_tlb_pending | |
52bf082f | 84 | __flush_tlb_pending: /* 26 insns */ |
1da177e4 LT |
85 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
86 | rdpr %pstate, %g7 | |
87 | sllx %o1, 3, %o1 | |
88 | andn %g7, PSTATE_IE, %g2 | |
89 | wrpr %g2, %pstate | |
90 | mov SECONDARY_CONTEXT, %o4 | |
91 | ldxa [%o4] ASI_DMMU, %g2 | |
92 | stxa %o0, [%o4] ASI_DMMU | |
93 | 1: sub %o1, (1 << 3), %o1 | |
94 | ldx [%o2 + %o1], %o3 | |
95 | andcc %o3, 1, %g0 | |
96 | andn %o3, 1, %o3 | |
97 | be,pn %icc, 2f | |
98 | or %o3, 0x10, %o3 | |
99 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
100 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
101 | membar #Sync | |
102 | brnz,pt %o1, 1b | |
103 | nop | |
104 | stxa %g2, [%o4] ASI_DMMU | |
4da808c3 DM |
105 | sethi %hi(KERNBASE), %o4 |
106 | flush %o4 | |
1da177e4 LT |
107 | retl |
108 | wrpr %g7, 0x0, %pstate | |
fef43da4 | 109 | nop |
2ef27778 DM |
110 | nop |
111 | nop | |
112 | nop | |
1da177e4 LT |
113 | |
114 | .align 32 | |
115 | .globl __flush_tlb_kernel_range | |
1daef08a | 116 | __flush_tlb_kernel_range: /* 16 insns */ |
52bf082f | 117 | /* %o0=start, %o1=end */ |
1da177e4 LT |
118 | cmp %o0, %o1 |
119 | be,pn %xcc, 2f | |
120 | sethi %hi(PAGE_SIZE), %o4 | |
121 | sub %o1, %o0, %o3 | |
122 | sub %o3, %o4, %o3 | |
123 | or %o0, 0x20, %o0 ! Nucleus | |
124 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP | |
125 | stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP | |
126 | membar #Sync | |
127 | brnz,pt %o3, 1b | |
128 | sub %o3, %o4, %o3 | |
4da808c3 DM |
129 | 2: sethi %hi(KERNBASE), %o3 |
130 | flush %o3 | |
131 | retl | |
132 | nop | |
52bf082f | 133 | nop |
1da177e4 LT |
134 | |
135 | __spitfire_flush_tlb_mm_slow: | |
136 | rdpr %pstate, %g1 | |
137 | wrpr %g1, PSTATE_IE, %pstate | |
138 | stxa %o0, [%o1] ASI_DMMU | |
139 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
140 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
141 | flush %g6 | |
142 | stxa %g2, [%o1] ASI_DMMU | |
4da808c3 DM |
143 | sethi %hi(KERNBASE), %o1 |
144 | flush %o1 | |
1da177e4 LT |
145 | retl |
146 | wrpr %g1, 0, %pstate | |
147 | ||
148 | /* | |
149 | * The following code flushes one page_size worth. | |
150 | */ | |
83005161 | 151 | .section .kprobes.text, "ax" |
1da177e4 LT |
152 | .align 32 |
153 | .globl __flush_icache_page | |
154 | __flush_icache_page: /* %o0 = phys_page */ | |
1da177e4 | 155 | srlx %o0, PAGE_SHIFT, %o0 |
b2d43834 | 156 | sethi %hi(PAGE_OFFSET), %g1 |
1da177e4 LT |
157 | sllx %o0, PAGE_SHIFT, %o0 |
158 | sethi %hi(PAGE_SIZE), %g2 | |
b2d43834 | 159 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
1da177e4 LT |
160 | add %o0, %g1, %o0 |
161 | 1: subcc %g2, 32, %g2 | |
162 | bne,pt %icc, 1b | |
163 | flush %o0 + %g2 | |
164 | retl | |
165 | nop | |
166 | ||
167 | #ifdef DCACHE_ALIASING_POSSIBLE | |
168 | ||
169 | #if (PAGE_SHIFT != 13) | |
170 | #error only page shift of 13 is supported by dcache flush | |
171 | #endif | |
172 | ||
173 | #define DTAG_MASK 0x3 | |
174 | ||
c5bd50a9 DM |
175 | /* This routine is Spitfire specific so the hardcoded |
176 | * D-cache size and line-size are OK. | |
177 | */ | |
1da177e4 LT |
178 | .align 64 |
179 | .globl __flush_dcache_page | |
180 | __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ | |
b2d43834 DM |
181 | sethi %hi(PAGE_OFFSET), %g1 |
182 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 | |
c5bd50a9 DM |
183 | sub %o0, %g1, %o0 ! physical address |
184 | srlx %o0, 11, %o0 ! make D-cache TAG | |
185 | sethi %hi(1 << 14), %o2 ! D-cache size | |
186 | sub %o2, (1 << 5), %o2 ! D-cache line size | |
187 | 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG | |
188 | andcc %o3, DTAG_MASK, %g0 ! Valid? | |
189 | be,pn %xcc, 2f ! Nope, branch | |
190 | andn %o3, DTAG_MASK, %o3 ! Clear valid bits | |
191 | cmp %o3, %o0 ! TAG match? | |
192 | bne,pt %xcc, 2f ! Nope, branch | |
193 | nop | |
194 | stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG | |
195 | membar #Sync | |
196 | 2: brnz,pt %o2, 1b | |
197 | sub %o2, (1 << 5), %o2 ! D-cache line size | |
1da177e4 LT |
198 | |
199 | /* The I-cache does not snoop local stores so we | |
200 | * better flush that too when necessary. | |
201 | */ | |
202 | brnz,pt %o1, __flush_icache_page | |
203 | sllx %o0, 11, %o0 | |
204 | retl | |
205 | nop | |
206 | ||
1da177e4 LT |
207 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
208 | ||
c5bd50a9 DM |
209 | .previous |
210 | ||
2ef27778 | 211 | /* Cheetah specific versions, patched at boot time. */ |
4da808c3 | 212 | __cheetah_flush_tlb_mm: /* 19 insns */ |
1da177e4 LT |
213 | rdpr %pstate, %g7 |
214 | andn %g7, PSTATE_IE, %g2 | |
215 | wrpr %g2, 0x0, %pstate | |
216 | wrpr %g0, 1, %tl | |
217 | mov PRIMARY_CONTEXT, %o2 | |
218 | mov 0x40, %g3 | |
219 | ldxa [%o2] ASI_DMMU, %g2 | |
2ef27778 DM |
220 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 |
221 | sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 | |
222 | or %o0, %o1, %o0 /* Preserve nucleus page size fields */ | |
1da177e4 LT |
223 | stxa %o0, [%o2] ASI_DMMU |
224 | stxa %g0, [%g3] ASI_DMMU_DEMAP | |
225 | stxa %g0, [%g3] ASI_IMMU_DEMAP | |
226 | stxa %g2, [%o2] ASI_DMMU | |
4da808c3 DM |
227 | sethi %hi(KERNBASE), %o2 |
228 | flush %o2 | |
1da177e4 LT |
229 | wrpr %g0, 0, %tl |
230 | retl | |
231 | wrpr %g7, 0x0, %pstate | |
232 | ||
f36391d2 DM |
233 | __cheetah_flush_tlb_page: /* 22 insns */ |
234 | /* %o0 = context, %o1 = vaddr */ | |
235 | rdpr %pstate, %g7 | |
236 | andn %g7, PSTATE_IE, %g2 | |
237 | wrpr %g2, 0x0, %pstate | |
238 | wrpr %g0, 1, %tl | |
239 | mov PRIMARY_CONTEXT, %o4 | |
240 | ldxa [%o4] ASI_DMMU, %g2 | |
241 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 | |
242 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 | |
243 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ | |
244 | stxa %o0, [%o4] ASI_DMMU | |
245 | andcc %o1, 1, %g0 | |
246 | be,pn %icc, 1f | |
247 | andn %o1, 1, %o3 | |
248 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
249 | 1: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
250 | membar #Sync | |
251 | stxa %g2, [%o4] ASI_DMMU | |
252 | sethi %hi(KERNBASE), %o4 | |
253 | flush %o4 | |
254 | wrpr %g0, 0, %tl | |
255 | retl | |
256 | wrpr %g7, 0x0, %pstate | |
257 | ||
4da808c3 | 258 | __cheetah_flush_tlb_pending: /* 27 insns */ |
1da177e4 LT |
259 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
260 | rdpr %pstate, %g7 | |
261 | sllx %o1, 3, %o1 | |
262 | andn %g7, PSTATE_IE, %g2 | |
263 | wrpr %g2, 0x0, %pstate | |
264 | wrpr %g0, 1, %tl | |
265 | mov PRIMARY_CONTEXT, %o4 | |
266 | ldxa [%o4] ASI_DMMU, %g2 | |
2ef27778 DM |
267 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 |
268 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 | |
269 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ | |
1da177e4 LT |
270 | stxa %o0, [%o4] ASI_DMMU |
271 | 1: sub %o1, (1 << 3), %o1 | |
272 | ldx [%o2 + %o1], %o3 | |
273 | andcc %o3, 1, %g0 | |
274 | be,pn %icc, 2f | |
275 | andn %o3, 1, %o3 | |
276 | stxa %g0, [%o3] ASI_IMMU_DEMAP | |
277 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP | |
b445e26c | 278 | membar #Sync |
1da177e4 | 279 | brnz,pt %o1, 1b |
b445e26c | 280 | nop |
1da177e4 | 281 | stxa %g2, [%o4] ASI_DMMU |
4da808c3 DM |
282 | sethi %hi(KERNBASE), %o4 |
283 | flush %o4 | |
1da177e4 LT |
284 | wrpr %g0, 0, %tl |
285 | retl | |
286 | wrpr %g7, 0x0, %pstate | |
287 | ||
288 | #ifdef DCACHE_ALIASING_POSSIBLE | |
c5bd50a9 | 289 | __cheetah_flush_dcache_page: /* 11 insns */ |
b2d43834 DM |
290 | sethi %hi(PAGE_OFFSET), %g1 |
291 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 | |
1da177e4 LT |
292 | sub %o0, %g1, %o0 |
293 | sethi %hi(PAGE_SIZE), %o4 | |
294 | 1: subcc %o4, (1 << 5), %o4 | |
295 | stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE | |
296 | membar #Sync | |
297 | bne,pt %icc, 1b | |
298 | nop | |
299 | retl /* I-cache flush never needed on Cheetah, see callers. */ | |
300 | nop | |
301 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
302 | ||
52bf082f | 303 | /* Hypervisor specific versions, patched at boot time. */ |
2a3a5f5d DM |
304 | __hypervisor_tlb_tl0_error: |
305 | save %sp, -192, %sp | |
306 | mov %i0, %o0 | |
307 | call hypervisor_tlbop_error | |
308 | mov %i1, %o1 | |
309 | ret | |
310 | restore | |
311 | ||
312 | __hypervisor_flush_tlb_mm: /* 10 insns */ | |
52bf082f DM |
313 | mov %o0, %o2 /* ARG2: mmu context */ |
314 | mov 0, %o0 /* ARG0: CPU lists unimplemented */ | |
315 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ | |
316 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ | |
317 | mov HV_FAST_MMU_DEMAP_CTX, %o5 | |
318 | ta HV_FAST_TRAP | |
2a3a5f5d DM |
319 | brnz,pn %o0, __hypervisor_tlb_tl0_error |
320 | mov HV_FAST_MMU_DEMAP_CTX, %o1 | |
52bf082f DM |
321 | retl |
322 | nop | |
323 | ||
f36391d2 DM |
324 | __hypervisor_flush_tlb_page: /* 11 insns */ |
325 | /* %o0 = context, %o1 = vaddr */ | |
326 | mov %o0, %g2 | |
327 | mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ | |
328 | mov %g2, %o1 /* ARG1: mmu context */ | |
329 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ | |
330 | srlx %o0, PAGE_SHIFT, %o0 | |
331 | sllx %o0, PAGE_SHIFT, %o0 | |
332 | ta HV_MMU_UNMAP_ADDR_TRAP | |
333 | brnz,pn %o0, __hypervisor_tlb_tl0_error | |
334 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 | |
335 | retl | |
336 | nop | |
337 | ||
2a3a5f5d | 338 | __hypervisor_flush_tlb_pending: /* 16 insns */ |
52bf082f DM |
339 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
340 | sllx %o1, 3, %g1 | |
341 | mov %o2, %g2 | |
342 | mov %o0, %g3 | |
343 | 1: sub %g1, (1 << 3), %g1 | |
344 | ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ | |
345 | mov %g3, %o1 /* ARG1: mmu context */ | |
2a3a5f5d DM |
346 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
347 | srlx %o0, PAGE_SHIFT, %o0 | |
348 | sllx %o0, PAGE_SHIFT, %o0 | |
52bf082f | 349 | ta HV_MMU_UNMAP_ADDR_TRAP |
2a3a5f5d DM |
350 | brnz,pn %o0, __hypervisor_tlb_tl0_error |
351 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 | |
52bf082f DM |
352 | brnz,pt %g1, 1b |
353 | nop | |
354 | retl | |
355 | nop | |
356 | ||
2a3a5f5d | 357 | __hypervisor_flush_tlb_kernel_range: /* 16 insns */ |
52bf082f DM |
358 | /* %o0=start, %o1=end */ |
359 | cmp %o0, %o1 | |
360 | be,pn %xcc, 2f | |
361 | sethi %hi(PAGE_SIZE), %g3 | |
362 | mov %o0, %g1 | |
363 | sub %o1, %g1, %g2 | |
364 | sub %g2, %g3, %g2 | |
365 | 1: add %g1, %g2, %o0 /* ARG0: virtual address */ | |
366 | mov 0, %o1 /* ARG1: mmu context */ | |
367 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ | |
368 | ta HV_MMU_UNMAP_ADDR_TRAP | |
2a3a5f5d DM |
369 | brnz,pn %o0, __hypervisor_tlb_tl0_error |
370 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 | |
52bf082f DM |
371 | brnz,pt %g2, 1b |
372 | sub %g2, %g3, %g2 | |
373 | 2: retl | |
374 | nop | |
375 | ||
376 | #ifdef DCACHE_ALIASING_POSSIBLE | |
377 | /* XXX Niagara and friends have an 8K cache, so no aliasing is | |
378 | * XXX possible, but nothing explicit in the Hypervisor API | |
379 | * XXX guarantees this. | |
380 | */ | |
381 | __hypervisor_flush_dcache_page: /* 2 insns */ | |
382 | retl | |
383 | nop | |
384 | #endif | |
385 | ||
386 | tlb_patch_one: | |
1da177e4 LT |
387 | 1: lduw [%o1], %g1 |
388 | stw %g1, [%o0] | |
389 | flush %o0 | |
390 | subcc %o2, 1, %o2 | |
391 | add %o1, 4, %o1 | |
392 | bne,pt %icc, 1b | |
393 | add %o0, 4, %o0 | |
394 | retl | |
395 | nop | |
396 | ||
397 | .globl cheetah_patch_cachetlbops | |
398 | cheetah_patch_cachetlbops: | |
399 | save %sp, -128, %sp | |
400 | ||
401 | sethi %hi(__flush_tlb_mm), %o0 | |
402 | or %o0, %lo(__flush_tlb_mm), %o0 | |
403 | sethi %hi(__cheetah_flush_tlb_mm), %o1 | |
404 | or %o1, %lo(__cheetah_flush_tlb_mm), %o1 | |
52bf082f | 405 | call tlb_patch_one |
4da808c3 | 406 | mov 19, %o2 |
1da177e4 | 407 | |
f36391d2 DM |
408 | sethi %hi(__flush_tlb_page), %o0 |
409 | or %o0, %lo(__flush_tlb_page), %o0 | |
410 | sethi %hi(__cheetah_flush_tlb_page), %o1 | |
411 | or %o1, %lo(__cheetah_flush_tlb_page), %o1 | |
412 | call tlb_patch_one | |
413 | mov 22, %o2 | |
414 | ||
1da177e4 LT |
415 | sethi %hi(__flush_tlb_pending), %o0 |
416 | or %o0, %lo(__flush_tlb_pending), %o0 | |
417 | sethi %hi(__cheetah_flush_tlb_pending), %o1 | |
418 | or %o1, %lo(__cheetah_flush_tlb_pending), %o1 | |
52bf082f | 419 | call tlb_patch_one |
4da808c3 | 420 | mov 27, %o2 |
1da177e4 LT |
421 | |
422 | #ifdef DCACHE_ALIASING_POSSIBLE | |
423 | sethi %hi(__flush_dcache_page), %o0 | |
424 | or %o0, %lo(__flush_dcache_page), %o0 | |
c5bd50a9 DM |
425 | sethi %hi(__cheetah_flush_dcache_page), %o1 |
426 | or %o1, %lo(__cheetah_flush_dcache_page), %o1 | |
52bf082f | 427 | call tlb_patch_one |
1da177e4 LT |
428 | mov 11, %o2 |
429 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
430 | ||
431 | ret | |
432 | restore | |
433 | ||
434 | #ifdef CONFIG_SMP | |
435 | /* These are all called by the slaves of a cross call, at | |
436 | * trap level 1, with interrupts fully disabled. | |
437 | * | |
438 | * Register usage: | |
439 | * %g5 mm->context (all tlb flushes) | |
440 | * %g1 address arg 1 (tlb page and range flushes) | |
441 | * %g7 address arg 2 (tlb range flush only) | |
442 | * | |
56fb4df6 DM |
443 | * %g6 scratch 1 |
444 | * %g2 scratch 2 | |
445 | * %g3 scratch 3 | |
446 | * %g4 scratch 4 | |
1da177e4 LT |
447 | */ |
448 | .align 32 | |
449 | .globl xcall_flush_tlb_mm | |
2a3a5f5d | 450 | xcall_flush_tlb_mm: /* 21 insns */ |
1da177e4 | 451 | mov PRIMARY_CONTEXT, %g2 |
1da177e4 | 452 | ldxa [%g2] ASI_DMMU, %g3 |
2ef27778 DM |
453 | srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 |
454 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 | |
455 | or %g5, %g4, %g5 /* Preserve nucleus page size fields */ | |
1da177e4 | 456 | stxa %g5, [%g2] ASI_DMMU |
2ef27778 | 457 | mov 0x40, %g4 |
1da177e4 LT |
458 | stxa %g0, [%g4] ASI_DMMU_DEMAP |
459 | stxa %g0, [%g4] ASI_IMMU_DEMAP | |
460 | stxa %g3, [%g2] ASI_DMMU | |
461 | retry | |
52bf082f DM |
462 | nop |
463 | nop | |
464 | nop | |
465 | nop | |
466 | nop | |
467 | nop | |
468 | nop | |
2a3a5f5d DM |
469 | nop |
470 | nop | |
471 | nop | |
1da177e4 | 472 | |
f36391d2 DM |
473 | .globl xcall_flush_tlb_page |
474 | xcall_flush_tlb_page: /* 17 insns */ | |
475 | /* %g5=context, %g1=vaddr */ | |
1da177e4 LT |
476 | mov PRIMARY_CONTEXT, %g4 |
477 | ldxa [%g4] ASI_DMMU, %g2 | |
2ef27778 DM |
478 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 |
479 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 | |
480 | or %g5, %g4, %g5 | |
481 | mov PRIMARY_CONTEXT, %g4 | |
1da177e4 | 482 | stxa %g5, [%g4] ASI_DMMU |
f36391d2 | 483 | andcc %g1, 0x1, %g0 |
1da177e4 | 484 | be,pn %icc, 2f |
f36391d2 | 485 | andn %g1, 0x1, %g5 |
1da177e4 LT |
486 | stxa %g0, [%g5] ASI_IMMU_DEMAP |
487 | 2: stxa %g0, [%g5] ASI_DMMU_DEMAP | |
488 | membar #Sync | |
1da177e4 LT |
489 | stxa %g2, [%g4] ASI_DMMU |
490 | retry | |
2a3a5f5d | 491 | nop |
f36391d2 | 492 | nop |
1da177e4 LT |
493 | |
494 | .globl xcall_flush_tlb_kernel_range | |
2a3a5f5d | 495 | xcall_flush_tlb_kernel_range: /* 25 insns */ |
1da177e4 LT |
496 | sethi %hi(PAGE_SIZE - 1), %g2 |
497 | or %g2, %lo(PAGE_SIZE - 1), %g2 | |
498 | andn %g1, %g2, %g1 | |
499 | andn %g7, %g2, %g7 | |
500 | sub %g7, %g1, %g3 | |
501 | add %g2, 1, %g2 | |
502 | sub %g3, %g2, %g3 | |
503 | or %g1, 0x20, %g1 ! Nucleus | |
504 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP | |
505 | stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP | |
506 | membar #Sync | |
507 | brnz,pt %g3, 1b | |
508 | sub %g3, %g2, %g3 | |
509 | retry | |
510 | nop | |
511 | nop | |
52bf082f DM |
512 | nop |
513 | nop | |
514 | nop | |
515 | nop | |
516 | nop | |
517 | nop | |
2a3a5f5d DM |
518 | nop |
519 | nop | |
520 | nop | |
1da177e4 LT |
521 | |
522 | /* This runs in a very controlled environment, so we do | |
523 | * not need to worry about BH races etc. | |
524 | */ | |
525 | .globl xcall_sync_tick | |
526 | xcall_sync_tick: | |
45fec05f DM |
527 | |
528 | 661: rdpr %pstate, %g2 | |
1da177e4 | 529 | wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate |
df7d6aec | 530 | .section .sun4v_2insn_patch, "ax" |
45fec05f DM |
531 | .word 661b |
532 | nop | |
533 | nop | |
534 | .previous | |
535 | ||
1da177e4 | 536 | rdpr %pil, %g2 |
b4f4372f | 537 | wrpr %g0, PIL_NORMAL_MAX, %pil |
1da177e4 LT |
538 | sethi %hi(109f), %g7 |
539 | b,pt %xcc, etrap_irq | |
540 | 109: or %g7, %lo(109b), %g7 | |
10e26723 DM |
541 | #ifdef CONFIG_TRACE_IRQFLAGS |
542 | call trace_hardirqs_off | |
543 | nop | |
544 | #endif | |
1da177e4 LT |
545 | call smp_synchronize_tick_client |
546 | nop | |
1da177e4 LT |
547 | b rtrap_xcall |
548 | ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 | |
549 | ||
93dae5b7 DM |
550 | .globl xcall_fetch_glob_regs |
551 | xcall_fetch_glob_regs: | |
916ca14a DM |
552 | sethi %hi(global_cpu_snapshot), %g1 |
553 | or %g1, %lo(global_cpu_snapshot), %g1 | |
93dae5b7 DM |
554 | __GET_CPUID(%g2) |
555 | sllx %g2, 6, %g3 | |
556 | add %g1, %g3, %g1 | |
557 | rdpr %tstate, %g7 | |
558 | stx %g7, [%g1 + GR_SNAP_TSTATE] | |
559 | rdpr %tpc, %g7 | |
560 | stx %g7, [%g1 + GR_SNAP_TPC] | |
561 | rdpr %tnpc, %g7 | |
562 | stx %g7, [%g1 + GR_SNAP_TNPC] | |
563 | stx %o7, [%g1 + GR_SNAP_O7] | |
564 | stx %i7, [%g1 + GR_SNAP_I7] | |
5afe2738 | 565 | /* Don't try this at home kids... */ |
a5a737e0 DM |
566 | rdpr %cwp, %g3 |
567 | sub %g3, 1, %g7 | |
5afe2738 DM |
568 | wrpr %g7, %cwp |
569 | mov %i7, %g7 | |
a5a737e0 | 570 | wrpr %g3, %cwp |
5afe2738 | 571 | stx %g7, [%g1 + GR_SNAP_RPC] |
93dae5b7 DM |
572 | sethi %hi(trap_block), %g7 |
573 | or %g7, %lo(trap_block), %g7 | |
574 | sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 | |
575 | add %g7, %g2, %g7 | |
576 | ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 | |
93dae5b7 DM |
577 | stx %g3, [%g1 + GR_SNAP_THREAD] |
578 | retry | |
93dae5b7 | 579 | |
916ca14a DM |
580 | .globl xcall_fetch_glob_pmu |
581 | xcall_fetch_glob_pmu: | |
582 | sethi %hi(global_cpu_snapshot), %g1 | |
583 | or %g1, %lo(global_cpu_snapshot), %g1 | |
584 | __GET_CPUID(%g2) | |
585 | sllx %g2, 6, %g3 | |
586 | add %g1, %g3, %g1 | |
587 | rd %pic, %g7 | |
588 | stx %g7, [%g1 + (4 * 8)] | |
589 | rd %pcr, %g7 | |
590 | stx %g7, [%g1 + (0 * 8)] | |
591 | retry | |
592 | ||
593 | .globl xcall_fetch_glob_pmu_n4 | |
594 | xcall_fetch_glob_pmu_n4: | |
595 | sethi %hi(global_cpu_snapshot), %g1 | |
596 | or %g1, %lo(global_cpu_snapshot), %g1 | |
597 | __GET_CPUID(%g2) | |
598 | sllx %g2, 6, %g3 | |
599 | add %g1, %g3, %g1 | |
600 | ||
601 | ldxa [%g0] ASI_PIC, %g7 | |
602 | stx %g7, [%g1 + (4 * 8)] | |
603 | mov 0x08, %g3 | |
604 | ldxa [%g3] ASI_PIC, %g7 | |
605 | stx %g7, [%g1 + (5 * 8)] | |
606 | mov 0x10, %g3 | |
607 | ldxa [%g3] ASI_PIC, %g7 | |
608 | stx %g7, [%g1 + (6 * 8)] | |
609 | mov 0x18, %g3 | |
610 | ldxa [%g3] ASI_PIC, %g7 | |
611 | stx %g7, [%g1 + (7 * 8)] | |
612 | ||
613 | mov %o0, %g2 | |
614 | mov %o1, %g3 | |
615 | mov %o5, %g7 | |
616 | ||
617 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
618 | mov 3, %o0 | |
619 | ta HV_FAST_TRAP | |
620 | stx %o1, [%g1 + (3 * 8)] | |
621 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
622 | mov 2, %o0 | |
623 | ta HV_FAST_TRAP | |
624 | stx %o1, [%g1 + (2 * 8)] | |
625 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
626 | mov 1, %o0 | |
627 | ta HV_FAST_TRAP | |
628 | stx %o1, [%g1 + (1 * 8)] | |
629 | mov HV_FAST_VT_GET_PERFREG, %o5 | |
630 | mov 0, %o0 | |
631 | ta HV_FAST_TRAP | |
632 | stx %o1, [%g1 + (0 * 8)] | |
633 | ||
634 | mov %g2, %o0 | |
635 | mov %g3, %o1 | |
636 | mov %g7, %o5 | |
637 | ||
638 | retry | |
639 | ||
1da177e4 LT |
640 | #ifdef DCACHE_ALIASING_POSSIBLE |
641 | .align 32 | |
642 | .globl xcall_flush_dcache_page_cheetah | |
643 | xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ | |
644 | sethi %hi(PAGE_SIZE), %g3 | |
645 | 1: subcc %g3, (1 << 5), %g3 | |
646 | stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE | |
647 | membar #Sync | |
648 | bne,pt %icc, 1b | |
649 | nop | |
650 | retry | |
651 | nop | |
652 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
653 | ||
654 | .globl xcall_flush_dcache_page_spitfire | |
655 | xcall_flush_dcache_page_spitfire: /* %g1 == physical page address | |
656 | %g7 == kernel page virtual address | |
657 | %g5 == (page->mapping != NULL) */ | |
658 | #ifdef DCACHE_ALIASING_POSSIBLE | |
659 | srlx %g1, (13 - 2), %g1 ! Form tag comparitor | |
660 | sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K | |
661 | sub %g3, (1 << 5), %g3 ! D$ linesize == 32 | |
662 | 1: ldxa [%g3] ASI_DCACHE_TAG, %g2 | |
663 | andcc %g2, 0x3, %g0 | |
664 | be,pn %xcc, 2f | |
665 | andn %g2, 0x3, %g2 | |
666 | cmp %g2, %g1 | |
667 | ||
668 | bne,pt %xcc, 2f | |
669 | nop | |
670 | stxa %g0, [%g3] ASI_DCACHE_TAG | |
671 | membar #Sync | |
672 | 2: cmp %g3, 0 | |
673 | bne,pt %xcc, 1b | |
674 | sub %g3, (1 << 5), %g3 | |
675 | ||
676 | brz,pn %g5, 2f | |
677 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
678 | sethi %hi(PAGE_SIZE), %g3 | |
679 | ||
680 | 1: flush %g7 | |
681 | subcc %g3, (1 << 5), %g3 | |
682 | bne,pt %icc, 1b | |
683 | add %g7, (1 << 5), %g7 | |
684 | ||
685 | 2: retry | |
686 | nop | |
687 | nop | |
688 | ||
2a3a5f5d DM |
689 | /* %g5: error |
690 | * %g6: tlb op | |
691 | */ | |
692 | __hypervisor_tlb_xcall_error: | |
693 | mov %g5, %g4 | |
694 | mov %g6, %g5 | |
695 | ba,pt %xcc, etrap | |
696 | rd %pc, %g7 | |
697 | mov %l4, %o0 | |
698 | call hypervisor_tlbop_error_xcall | |
699 | mov %l5, %o1 | |
7697daaa | 700 | ba,a,pt %xcc, rtrap |
2a3a5f5d | 701 | |
52bf082f | 702 | .globl __hypervisor_xcall_flush_tlb_mm |
2a3a5f5d | 703 | __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ |
52bf082f DM |
704 | /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ |
705 | mov %o0, %g2 | |
706 | mov %o1, %g3 | |
707 | mov %o2, %g4 | |
708 | mov %o3, %g1 | |
709 | mov %o5, %g7 | |
710 | clr %o0 /* ARG0: CPU lists unimplemented */ | |
711 | clr %o1 /* ARG1: CPU lists unimplemented */ | |
712 | mov %g5, %o2 /* ARG2: mmu context */ | |
713 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ | |
714 | mov HV_FAST_MMU_DEMAP_CTX, %o5 | |
715 | ta HV_FAST_TRAP | |
2a3a5f5d DM |
716 | mov HV_FAST_MMU_DEMAP_CTX, %g6 |
717 | brnz,pn %o0, __hypervisor_tlb_xcall_error | |
718 | mov %o0, %g5 | |
52bf082f DM |
719 | mov %g2, %o0 |
720 | mov %g3, %o1 | |
721 | mov %g4, %o2 | |
722 | mov %g1, %o3 | |
723 | mov %g7, %o5 | |
724 | membar #Sync | |
725 | retry | |
726 | ||
f36391d2 DM |
727 | .globl __hypervisor_xcall_flush_tlb_page |
728 | __hypervisor_xcall_flush_tlb_page: /* 17 insns */ | |
729 | /* %g5=ctx, %g1=vaddr */ | |
52bf082f DM |
730 | mov %o0, %g2 |
731 | mov %o1, %g3 | |
732 | mov %o2, %g4 | |
f36391d2 | 733 | mov %g1, %o0 /* ARG0: virtual address */ |
52bf082f | 734 | mov %g5, %o1 /* ARG1: mmu context */ |
2a3a5f5d DM |
735 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
736 | srlx %o0, PAGE_SHIFT, %o0 | |
737 | sllx %o0, PAGE_SHIFT, %o0 | |
52bf082f | 738 | ta HV_MMU_UNMAP_ADDR_TRAP |
2a3a5f5d DM |
739 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
740 | brnz,a,pn %o0, __hypervisor_tlb_xcall_error | |
741 | mov %o0, %g5 | |
52bf082f DM |
742 | mov %g2, %o0 |
743 | mov %g3, %o1 | |
744 | mov %g4, %o2 | |
745 | membar #Sync | |
746 | retry | |
747 | ||
748 | .globl __hypervisor_xcall_flush_tlb_kernel_range | |
2a3a5f5d DM |
749 | __hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ |
750 | /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ | |
52bf082f DM |
751 | sethi %hi(PAGE_SIZE - 1), %g2 |
752 | or %g2, %lo(PAGE_SIZE - 1), %g2 | |
753 | andn %g1, %g2, %g1 | |
754 | andn %g7, %g2, %g7 | |
755 | sub %g7, %g1, %g3 | |
756 | add %g2, 1, %g2 | |
757 | sub %g3, %g2, %g3 | |
758 | mov %o0, %g2 | |
759 | mov %o1, %g4 | |
2a3a5f5d | 760 | mov %o2, %g7 |
52bf082f DM |
761 | 1: add %g1, %g3, %o0 /* ARG0: virtual address */ |
762 | mov 0, %o1 /* ARG1: mmu context */ | |
763 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ | |
764 | ta HV_MMU_UNMAP_ADDR_TRAP | |
2a3a5f5d DM |
765 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
766 | brnz,pn %o0, __hypervisor_tlb_xcall_error | |
767 | mov %o0, %g5 | |
52bf082f DM |
768 | sethi %hi(PAGE_SIZE), %o2 |
769 | brnz,pt %g3, 1b | |
770 | sub %g3, %o2, %g3 | |
771 | mov %g2, %o0 | |
772 | mov %g4, %o1 | |
2a3a5f5d | 773 | mov %g7, %o2 |
52bf082f DM |
774 | membar #Sync |
775 | retry | |
776 | ||
1da177e4 LT |
777 | /* These just get rescheduled to PIL vectors. */ |
778 | .globl xcall_call_function | |
779 | xcall_call_function: | |
780 | wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint | |
781 | retry | |
782 | ||
d172ad18 DM |
783 | .globl xcall_call_function_single |
784 | xcall_call_function_single: | |
785 | wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint | |
786 | retry | |
787 | ||
1da177e4 LT |
788 | .globl xcall_receive_signal |
789 | xcall_receive_signal: | |
790 | wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint | |
791 | retry | |
792 | ||
793 | .globl xcall_capture | |
794 | xcall_capture: | |
795 | wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint | |
796 | retry | |
797 | ||
ee29074d DM |
798 | .globl xcall_new_mmu_context_version |
799 | xcall_new_mmu_context_version: | |
800 | wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint | |
801 | retry | |
802 | ||
e2fdd7fd DM |
803 | #ifdef CONFIG_KGDB |
804 | .globl xcall_kgdb_capture | |
805 | xcall_kgdb_capture: | |
42cc77c8 DM |
806 | wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint |
807 | retry | |
e2fdd7fd DM |
808 | #endif |
809 | ||
1da177e4 | 810 | #endif /* CONFIG_SMP */ |
52bf082f DM |
811 | |
812 | ||
813 | .globl hypervisor_patch_cachetlbops | |
814 | hypervisor_patch_cachetlbops: | |
815 | save %sp, -128, %sp | |
816 | ||
817 | sethi %hi(__flush_tlb_mm), %o0 | |
818 | or %o0, %lo(__flush_tlb_mm), %o0 | |
819 | sethi %hi(__hypervisor_flush_tlb_mm), %o1 | |
820 | or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 | |
821 | call tlb_patch_one | |
2a3a5f5d | 822 | mov 10, %o2 |
52bf082f | 823 | |
f36391d2 DM |
824 | sethi %hi(__flush_tlb_page), %o0 |
825 | or %o0, %lo(__flush_tlb_page), %o0 | |
826 | sethi %hi(__hypervisor_flush_tlb_page), %o1 | |
827 | or %o1, %lo(__hypervisor_flush_tlb_page), %o1 | |
828 | call tlb_patch_one | |
829 | mov 11, %o2 | |
830 | ||
52bf082f DM |
831 | sethi %hi(__flush_tlb_pending), %o0 |
832 | or %o0, %lo(__flush_tlb_pending), %o0 | |
833 | sethi %hi(__hypervisor_flush_tlb_pending), %o1 | |
834 | or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 | |
835 | call tlb_patch_one | |
2a3a5f5d | 836 | mov 16, %o2 |
52bf082f DM |
837 | |
838 | sethi %hi(__flush_tlb_kernel_range), %o0 | |
839 | or %o0, %lo(__flush_tlb_kernel_range), %o0 | |
840 | sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 | |
841 | or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 | |
842 | call tlb_patch_one | |
2a3a5f5d | 843 | mov 16, %o2 |
52bf082f DM |
844 | |
845 | #ifdef DCACHE_ALIASING_POSSIBLE | |
846 | sethi %hi(__flush_dcache_page), %o0 | |
847 | or %o0, %lo(__flush_dcache_page), %o0 | |
848 | sethi %hi(__hypervisor_flush_dcache_page), %o1 | |
849 | or %o1, %lo(__hypervisor_flush_dcache_page), %o1 | |
850 | call tlb_patch_one | |
851 | mov 2, %o2 | |
852 | #endif /* DCACHE_ALIASING_POSSIBLE */ | |
853 | ||
854 | #ifdef CONFIG_SMP | |
855 | sethi %hi(xcall_flush_tlb_mm), %o0 | |
856 | or %o0, %lo(xcall_flush_tlb_mm), %o0 | |
857 | sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 | |
858 | or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 | |
859 | call tlb_patch_one | |
2a3a5f5d | 860 | mov 21, %o2 |
52bf082f | 861 | |
f36391d2 DM |
862 | sethi %hi(xcall_flush_tlb_page), %o0 |
863 | or %o0, %lo(xcall_flush_tlb_page), %o0 | |
864 | sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 | |
865 | or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 | |
52bf082f | 866 | call tlb_patch_one |
f36391d2 | 867 | mov 17, %o2 |
52bf082f DM |
868 | |
869 | sethi %hi(xcall_flush_tlb_kernel_range), %o0 | |
870 | or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 | |
871 | sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 | |
872 | or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 | |
873 | call tlb_patch_one | |
2a3a5f5d | 874 | mov 25, %o2 |
52bf082f DM |
875 | #endif /* CONFIG_SMP */ |
876 | ||
877 | ret | |
878 | restore |