projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'platform-drivers-x86-v4.7-1' of git://git.infradead.org/users/dvhart/linux...
[deliverable/linux.git] / arch / sparc / mm / ultra.S
CommitLineData
b00dc837 1/*
1da177e4
LT		 2 * ultra.S: Don't expand these all over the place...
3 *
93dae5b7 4 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
1da177e4
LT		 5 */
6
1da177e4
LT		 7#include <asm/asi.h>
8#include <asm/pgtable.h>
9#include <asm/page.h>
10#include <asm/spitfire.h>
11#include <asm/mmu_context.h>
2ef27778 12#include <asm/mmu.h>
1da177e4
LT		 13#include <asm/pil.h>
14#include <asm/head.h>
15#include <asm/thread_info.h>
16#include <asm/cacheflush.h>
52bf082f 17#include <asm/hypervisor.h>
93dae5b7 18#include <asm/cpudata.h>
1da177e4
LT		 19
20 /* Basically, most of the Spitfire vs. Cheetah madness
21 * has to do with the fact that Cheetah does not support
22 * IMMU flushes out of the secondary context. Someone needs
23 * to throw a south lake birthday party for the folks
24 * in Microelectronics who refused to fix this shit.
25 */
26
27 /* This file is meant to be read efficiently by the CPU, not humans.
28 * Staraj sie tego nikomu nie pierdolnac...
29 */
30 .text
31 .align 32
32 .globl __flush_tlb_mm
52bf082f
DM		 33__flush_tlb_mm: /* 18 insns */
34 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
1da177e4
LT		 35 ldxa [%o1] ASI_DMMU, %g2
36 cmp %g2, %o0
37 bne,pn %icc, __spitfire_flush_tlb_mm_slow
38 mov 0x50, %g3
39 stxa %g0, [%g3] ASI_DMMU_DEMAP
40 stxa %g0, [%g3] ASI_IMMU_DEMAP
4da808c3
DM		 41 sethi %hi(KERNBASE), %g3
42 flush %g3
1da177e4 43 retl
4da808c3 44 nop
1da177e4
LT		 45 nop
46 nop
47 nop
48 nop
49 nop
50 nop
51 nop
2ef27778
DM		 52 nop
53 nop
1da177e4 54
f36391d2
DM		 55 .align 32
56 .globl __flush_tlb_page
57__flush_tlb_page: /* 22 insns */
58 /* %o0 = context, %o1 = vaddr */
59 rdpr %pstate, %g7
60 andn %g7, PSTATE_IE, %g2
61 wrpr %g2, %pstate
62 mov SECONDARY_CONTEXT, %o4
63 ldxa [%o4] ASI_DMMU, %g2
64 stxa %o0, [%o4] ASI_DMMU
65 andcc %o1, 1, %g0
66 andn %o1, 1, %o3
67 be,pn %icc, 1f
68 or %o3, 0x10, %o3
69 stxa %g0, [%o3] ASI_IMMU_DEMAP
701: stxa %g0, [%o3] ASI_DMMU_DEMAP
71 membar #Sync
72 stxa %g2, [%o4] ASI_DMMU
73 sethi %hi(KERNBASE), %o4
74 flush %o4
75 retl
76 wrpr %g7, 0x0, %pstate
77 nop
78 nop
79 nop
80 nop
81
1da177e4
LT		 82 .align 32
83 .globl __flush_tlb_pending
52bf082f 84__flush_tlb_pending: /* 26 insns */
1da177e4
LT		 85 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
86 rdpr %pstate, %g7
87 sllx %o1, 3, %o1
88 andn %g7, PSTATE_IE, %g2
89 wrpr %g2, %pstate
90 mov SECONDARY_CONTEXT, %o4
91 ldxa [%o4] ASI_DMMU, %g2
92 stxa %o0, [%o4] ASI_DMMU
931: sub %o1, (1 << 3), %o1
94 ldx [%o2 + %o1], %o3
95 andcc %o3, 1, %g0
96 andn %o3, 1, %o3
97 be,pn %icc, 2f
98 or %o3, 0x10, %o3
99 stxa %g0, [%o3] ASI_IMMU_DEMAP
1002: stxa %g0, [%o3] ASI_DMMU_DEMAP
101 membar #Sync
102 brnz,pt %o1, 1b
103 nop
104 stxa %g2, [%o4] ASI_DMMU
4da808c3
DM		 105 sethi %hi(KERNBASE), %o4
106 flush %o4
1da177e4
LT		 107 retl
108 wrpr %g7, 0x0, %pstate
fef43da4 109 nop
2ef27778
DM		 110 nop
111 nop
112 nop
1da177e4
LT		 113
114 .align 32
115 .globl __flush_tlb_kernel_range
1daef08a 116__flush_tlb_kernel_range: /* 16 insns */
52bf082f 117 /* %o0=start, %o1=end */
1da177e4
LT		 118 cmp %o0, %o1
119 be,pn %xcc, 2f
120 sethi %hi(PAGE_SIZE), %o4
121 sub %o1, %o0, %o3
122 sub %o3, %o4, %o3
123 or %o0, 0x20, %o0 ! Nucleus
1241: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
125 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
126 membar #Sync
127 brnz,pt %o3, 1b
128 sub %o3, %o4, %o3
4da808c3
DM		 1292: sethi %hi(KERNBASE), %o3
130 flush %o3
131 retl
132 nop
52bf082f 133 nop
1da177e4
LT		 134
135__spitfire_flush_tlb_mm_slow:
136 rdpr %pstate, %g1
137 wrpr %g1, PSTATE_IE, %pstate
138 stxa %o0, [%o1] ASI_DMMU
139 stxa %g0, [%g3] ASI_DMMU_DEMAP
140 stxa %g0, [%g3] ASI_IMMU_DEMAP
141 flush %g6
142 stxa %g2, [%o1] ASI_DMMU
4da808c3
DM		 143 sethi %hi(KERNBASE), %o1
144 flush %o1
1da177e4
LT		 145 retl
146 wrpr %g1, 0, %pstate
147
148/*
149 * The following code flushes one page_size worth.
150 */
83005161 151 .section .kprobes.text, "ax"
1da177e4
LT		 152 .align 32
153 .globl __flush_icache_page
154__flush_icache_page: /* %o0 = phys_page */
1da177e4 155 srlx %o0, PAGE_SHIFT, %o0
b2d43834 156 sethi %hi(PAGE_OFFSET), %g1
1da177e4
LT		 157 sllx %o0, PAGE_SHIFT, %o0
158 sethi %hi(PAGE_SIZE), %g2
b2d43834 159 ldx [%g1 + %lo(PAGE_OFFSET)], %g1
1da177e4
LT		 160 add %o0, %g1, %o0
1611: subcc %g2, 32, %g2
162 bne,pt %icc, 1b
163 flush %o0 + %g2
164 retl
165 nop
166
167#ifdef DCACHE_ALIASING_POSSIBLE
168
169#if (PAGE_SHIFT != 13)
170#error only page shift of 13 is supported by dcache flush
171#endif
172
173#define DTAG_MASK 0x3
174
c5bd50a9
DM		 175 /* This routine is Spitfire specific so the hardcoded
176 * D-cache size and line-size are OK.
177 */
1da177e4
LT		 178 .align 64
179 .globl __flush_dcache_page
180__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
b2d43834
DM		 181 sethi %hi(PAGE_OFFSET), %g1
182 ldx [%g1 + %lo(PAGE_OFFSET)], %g1
c5bd50a9
DM		 183 sub %o0, %g1, %o0 ! physical address
184 srlx %o0, 11, %o0 ! make D-cache TAG
185 sethi %hi(1 << 14), %o2 ! D-cache size
186 sub %o2, (1 << 5), %o2 ! D-cache line size
1871: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG
188 andcc %o3, DTAG_MASK, %g0 ! Valid?
189 be,pn %xcc, 2f ! Nope, branch
190 andn %o3, DTAG_MASK, %o3 ! Clear valid bits
191 cmp %o3, %o0 ! TAG match?
192 bne,pt %xcc, 2f ! Nope, branch
193 nop
194 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG
195 membar #Sync
1962: brnz,pt %o2, 1b
197 sub %o2, (1 << 5), %o2 ! D-cache line size
1da177e4
LT		 198
199 /* The I-cache does not snoop local stores so we
200 * better flush that too when necessary.
201 */
202 brnz,pt %o1, __flush_icache_page
203 sllx %o0, 11, %o0
204 retl
205 nop
206
1da177e4
LT		 207#endif /* DCACHE_ALIASING_POSSIBLE */
208
c5bd50a9
DM		 209 .previous
210
2ef27778 211 /* Cheetah specific versions, patched at boot time. */
4da808c3 212__cheetah_flush_tlb_mm: /* 19 insns */
1da177e4
LT		 213 rdpr %pstate, %g7
214 andn %g7, PSTATE_IE, %g2
215 wrpr %g2, 0x0, %pstate
216 wrpr %g0, 1, %tl
217 mov PRIMARY_CONTEXT, %o2
218 mov 0x40, %g3
219 ldxa [%o2] ASI_DMMU, %g2
2ef27778
DM		 220 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1
221 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1
222 or %o0, %o1, %o0 /* Preserve nucleus page size fields */
1da177e4
LT		 223 stxa %o0, [%o2] ASI_DMMU
224 stxa %g0, [%g3] ASI_DMMU_DEMAP
225 stxa %g0, [%g3] ASI_IMMU_DEMAP
226 stxa %g2, [%o2] ASI_DMMU
4da808c3
DM		 227 sethi %hi(KERNBASE), %o2
228 flush %o2
1da177e4
LT		 229 wrpr %g0, 0, %tl
230 retl
231 wrpr %g7, 0x0, %pstate
232
f36391d2
DM		 233__cheetah_flush_tlb_page: /* 22 insns */
234 /* %o0 = context, %o1 = vaddr */
235 rdpr %pstate, %g7
236 andn %g7, PSTATE_IE, %g2
237 wrpr %g2, 0x0, %pstate
238 wrpr %g0, 1, %tl
239 mov PRIMARY_CONTEXT, %o4
240 ldxa [%o4] ASI_DMMU, %g2
241 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
242 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
243 or %o0, %o3, %o0 /* Preserve nucleus page size fields */
244 stxa %o0, [%o4] ASI_DMMU
245 andcc %o1, 1, %g0
246 be,pn %icc, 1f
247 andn %o1, 1, %o3
248 stxa %g0, [%o3] ASI_IMMU_DEMAP
2491: stxa %g0, [%o3] ASI_DMMU_DEMAP
250 membar #Sync
251 stxa %g2, [%o4] ASI_DMMU
252 sethi %hi(KERNBASE), %o4
253 flush %o4
254 wrpr %g0, 0, %tl
255 retl
256 wrpr %g7, 0x0, %pstate
257
4da808c3 258__cheetah_flush_tlb_pending: /* 27 insns */
1da177e4
LT		 259 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
260 rdpr %pstate, %g7
261 sllx %o1, 3, %o1
262 andn %g7, PSTATE_IE, %g2
263 wrpr %g2, 0x0, %pstate
264 wrpr %g0, 1, %tl
265 mov PRIMARY_CONTEXT, %o4
266 ldxa [%o4] ASI_DMMU, %g2
2ef27778
DM		 267 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
268 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
269 or %o0, %o3, %o0 /* Preserve nucleus page size fields */
1da177e4
LT		 270 stxa %o0, [%o4] ASI_DMMU
2711: sub %o1, (1 << 3), %o1
272 ldx [%o2 + %o1], %o3
273 andcc %o3, 1, %g0
274 be,pn %icc, 2f
275 andn %o3, 1, %o3
276 stxa %g0, [%o3] ASI_IMMU_DEMAP
2772: stxa %g0, [%o3] ASI_DMMU_DEMAP
b445e26c 278 membar #Sync
1da177e4 279 brnz,pt %o1, 1b
b445e26c 280 nop
1da177e4 281 stxa %g2, [%o4] ASI_DMMU
4da808c3
DM		 282 sethi %hi(KERNBASE), %o4
283 flush %o4
1da177e4
LT		 284 wrpr %g0, 0, %tl
285 retl
286 wrpr %g7, 0x0, %pstate
287
288#ifdef DCACHE_ALIASING_POSSIBLE
c5bd50a9 289__cheetah_flush_dcache_page: /* 11 insns */
b2d43834
DM		 290 sethi %hi(PAGE_OFFSET), %g1
291 ldx [%g1 + %lo(PAGE_OFFSET)], %g1
1da177e4
LT		 292 sub %o0, %g1, %o0
293 sethi %hi(PAGE_SIZE), %o4
2941: subcc %o4, (1 << 5), %o4
295 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
296 membar #Sync
297 bne,pt %icc, 1b
298 nop
299 retl /* I-cache flush never needed on Cheetah, see callers. */
300 nop
301#endif /* DCACHE_ALIASING_POSSIBLE */
302
52bf082f 303 /* Hypervisor specific versions, patched at boot time. */
2a3a5f5d
DM		 304__hypervisor_tlb_tl0_error:
305 save %sp, -192, %sp
306 mov %i0, %o0
307 call hypervisor_tlbop_error
308 mov %i1, %o1
309 ret
310 restore
311
312__hypervisor_flush_tlb_mm: /* 10 insns */
52bf082f
DM		 313 mov %o0, %o2 /* ARG2: mmu context */
314 mov 0, %o0 /* ARG0: CPU lists unimplemented */
315 mov 0, %o1 /* ARG1: CPU lists unimplemented */
316 mov HV_MMU_ALL, %o3 /* ARG3: flags */
317 mov HV_FAST_MMU_DEMAP_CTX, %o5
318 ta HV_FAST_TRAP
2a3a5f5d
DM		 319 brnz,pn %o0, __hypervisor_tlb_tl0_error
320 mov HV_FAST_MMU_DEMAP_CTX, %o1
52bf082f
DM		 321 retl
322 nop
323
f36391d2
DM		 324__hypervisor_flush_tlb_page: /* 11 insns */
325 /* %o0 = context, %o1 = vaddr */
326 mov %o0, %g2
327 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */
328 mov %g2, %o1 /* ARG1: mmu context */
329 mov HV_MMU_ALL, %o2 /* ARG2: flags */
330 srlx %o0, PAGE_SHIFT, %o0
331 sllx %o0, PAGE_SHIFT, %o0
332 ta HV_MMU_UNMAP_ADDR_TRAP
333 brnz,pn %o0, __hypervisor_tlb_tl0_error
334 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
335 retl
336 nop
337
2a3a5f5d 338__hypervisor_flush_tlb_pending: /* 16 insns */
52bf082f
DM		 339 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
340 sllx %o1, 3, %g1
341 mov %o2, %g2
342 mov %o0, %g3
3431: sub %g1, (1 << 3), %g1
344 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */
345 mov %g3, %o1 /* ARG1: mmu context */
2a3a5f5d
DM		 346 mov HV_MMU_ALL, %o2 /* ARG2: flags */
347 srlx %o0, PAGE_SHIFT, %o0
348 sllx %o0, PAGE_SHIFT, %o0
52bf082f 349 ta HV_MMU_UNMAP_ADDR_TRAP
2a3a5f5d
DM		 350 brnz,pn %o0, __hypervisor_tlb_tl0_error
351 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
52bf082f
DM		 352 brnz,pt %g1, 1b
353 nop
354 retl
355 nop
356
2a3a5f5d 357__hypervisor_flush_tlb_kernel_range: /* 16 insns */
52bf082f
DM		 358 /* %o0=start, %o1=end */
359 cmp %o0, %o1
360 be,pn %xcc, 2f
361 sethi %hi(PAGE_SIZE), %g3
362 mov %o0, %g1
363 sub %o1, %g1, %g2
364 sub %g2, %g3, %g2
3651: add %g1, %g2, %o0 /* ARG0: virtual address */
366 mov 0, %o1 /* ARG1: mmu context */
367 mov HV_MMU_ALL, %o2 /* ARG2: flags */
368 ta HV_MMU_UNMAP_ADDR_TRAP
2a3a5f5d
DM		 369 brnz,pn %o0, __hypervisor_tlb_tl0_error
370 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
52bf082f
DM		 371 brnz,pt %g2, 1b
372 sub %g2, %g3, %g2
3732: retl
374 nop
375
376#ifdef DCACHE_ALIASING_POSSIBLE
377 /* XXX Niagara and friends have an 8K cache, so no aliasing is
378 * XXX possible, but nothing explicit in the Hypervisor API
379 * XXX guarantees this.
380 */
381__hypervisor_flush_dcache_page: /* 2 insns */
382 retl
383 nop
384#endif
385
386tlb_patch_one:
1da177e4
LT		 3871: lduw [%o1], %g1
388 stw %g1, [%o0]
389 flush %o0
390 subcc %o2, 1, %o2
391 add %o1, 4, %o1
392 bne,pt %icc, 1b
393 add %o0, 4, %o0
394 retl
395 nop
396
397 .globl cheetah_patch_cachetlbops
398cheetah_patch_cachetlbops:
399 save %sp, -128, %sp
400
401 sethi %hi(__flush_tlb_mm), %o0
402 or %o0, %lo(__flush_tlb_mm), %o0
403 sethi %hi(__cheetah_flush_tlb_mm), %o1
404 or %o1, %lo(__cheetah_flush_tlb_mm), %o1
52bf082f 405 call tlb_patch_one
4da808c3 406 mov 19, %o2
1da177e4 407
f36391d2
DM		 408 sethi %hi(__flush_tlb_page), %o0
409 or %o0, %lo(__flush_tlb_page), %o0
410 sethi %hi(__cheetah_flush_tlb_page), %o1
411 or %o1, %lo(__cheetah_flush_tlb_page), %o1
412 call tlb_patch_one
413 mov 22, %o2
414
1da177e4
LT		 415 sethi %hi(__flush_tlb_pending), %o0
416 or %o0, %lo(__flush_tlb_pending), %o0
417 sethi %hi(__cheetah_flush_tlb_pending), %o1
418 or %o1, %lo(__cheetah_flush_tlb_pending), %o1
52bf082f 419 call tlb_patch_one
4da808c3 420 mov 27, %o2
1da177e4
LT		 421
422#ifdef DCACHE_ALIASING_POSSIBLE
423 sethi %hi(__flush_dcache_page), %o0
424 or %o0, %lo(__flush_dcache_page), %o0
c5bd50a9
DM		 425 sethi %hi(__cheetah_flush_dcache_page), %o1
426 or %o1, %lo(__cheetah_flush_dcache_page), %o1
52bf082f 427 call tlb_patch_one
1da177e4
LT		 428 mov 11, %o2
429#endif /* DCACHE_ALIASING_POSSIBLE */
430
431 ret
432 restore
433
434#ifdef CONFIG_SMP
435 /* These are all called by the slaves of a cross call, at
436 * trap level 1, with interrupts fully disabled.
437 *
438 * Register usage:
439 * %g5 mm->context (all tlb flushes)
440 * %g1 address arg 1 (tlb page and range flushes)
441 * %g7 address arg 2 (tlb range flush only)
442 *
56fb4df6
DM		 443 * %g6 scratch 1
444 * %g2 scratch 2
445 * %g3 scratch 3
446 * %g4 scratch 4
1da177e4
LT		 447 */
448 .align 32
449 .globl xcall_flush_tlb_mm
2a3a5f5d 450xcall_flush_tlb_mm: /* 21 insns */
1da177e4 451 mov PRIMARY_CONTEXT, %g2
1da177e4 452 ldxa [%g2] ASI_DMMU, %g3
2ef27778
DM		 453 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
454 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
455 or %g5, %g4, %g5 /* Preserve nucleus page size fields */
1da177e4 456 stxa %g5, [%g2] ASI_DMMU
2ef27778 457 mov 0x40, %g4
1da177e4
LT		 458 stxa %g0, [%g4] ASI_DMMU_DEMAP
459 stxa %g0, [%g4] ASI_IMMU_DEMAP
460 stxa %g3, [%g2] ASI_DMMU
461 retry
52bf082f
DM		 462 nop
463 nop
464 nop
465 nop
466 nop
467 nop
468 nop
2a3a5f5d
DM		 469 nop
470 nop
471 nop
1da177e4 472
f36391d2
DM		 473 .globl xcall_flush_tlb_page
474xcall_flush_tlb_page: /* 17 insns */
475 /* %g5=context, %g1=vaddr */
1da177e4
LT		 476 mov PRIMARY_CONTEXT, %g4
477 ldxa [%g4] ASI_DMMU, %g2
2ef27778
DM		 478 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4
479 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
480 or %g5, %g4, %g5
481 mov PRIMARY_CONTEXT, %g4
1da177e4 482 stxa %g5, [%g4] ASI_DMMU
f36391d2 483 andcc %g1, 0x1, %g0
1da177e4 484 be,pn %icc, 2f
f36391d2 485 andn %g1, 0x1, %g5
1da177e4
LT		 486 stxa %g0, [%g5] ASI_IMMU_DEMAP
4872: stxa %g0, [%g5] ASI_DMMU_DEMAP
488 membar #Sync
1da177e4
LT		 489 stxa %g2, [%g4] ASI_DMMU
490 retry
2a3a5f5d 491 nop
f36391d2 492 nop
1da177e4
LT		 493
494 .globl xcall_flush_tlb_kernel_range
2a3a5f5d 495xcall_flush_tlb_kernel_range: /* 25 insns */
1da177e4
LT		 496 sethi %hi(PAGE_SIZE - 1), %g2
497 or %g2, %lo(PAGE_SIZE - 1), %g2
498 andn %g1, %g2, %g1
499 andn %g7, %g2, %g7
500 sub %g7, %g1, %g3
501 add %g2, 1, %g2
502 sub %g3, %g2, %g3
503 or %g1, 0x20, %g1 ! Nucleus
5041: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
505 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
506 membar #Sync
507 brnz,pt %g3, 1b
508 sub %g3, %g2, %g3
509 retry
510 nop
511 nop
52bf082f
DM		 512 nop
513 nop
514 nop
515 nop
516 nop
517 nop
2a3a5f5d
DM		 518 nop
519 nop
520 nop
1da177e4
LT		 521
522 /* This runs in a very controlled environment, so we do
523 * not need to worry about BH races etc.
524 */
525 .globl xcall_sync_tick
526xcall_sync_tick:
45fec05f
DM		 527
528661: rdpr %pstate, %g2
1da177e4 529 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
df7d6aec 530 .section .sun4v_2insn_patch, "ax"
45fec05f
DM		 531 .word 661b
532 nop
533 nop
534 .previous
535
1da177e4 536 rdpr %pil, %g2
b4f4372f 537 wrpr %g0, PIL_NORMAL_MAX, %pil
1da177e4
LT		 538 sethi %hi(109f), %g7
539 b,pt %xcc, etrap_irq
540109: or %g7, %lo(109b), %g7
10e26723
DM		 541#ifdef CONFIG_TRACE_IRQFLAGS
542 call trace_hardirqs_off
543 nop
544#endif
1da177e4
LT		 545 call smp_synchronize_tick_client
546 nop
1da177e4
LT		 547 b rtrap_xcall
548 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
549
93dae5b7
DM		 550 .globl xcall_fetch_glob_regs
551xcall_fetch_glob_regs:
916ca14a
DM		 552 sethi %hi(global_cpu_snapshot), %g1
553 or %g1, %lo(global_cpu_snapshot), %g1
93dae5b7
DM		 554 __GET_CPUID(%g2)
555 sllx %g2, 6, %g3
556 add %g1, %g3, %g1
557 rdpr %tstate, %g7
558 stx %g7, [%g1 + GR_SNAP_TSTATE]
559 rdpr %tpc, %g7
560 stx %g7, [%g1 + GR_SNAP_TPC]
561 rdpr %tnpc, %g7
562 stx %g7, [%g1 + GR_SNAP_TNPC]
563 stx %o7, [%g1 + GR_SNAP_O7]
564 stx %i7, [%g1 + GR_SNAP_I7]
5afe2738 565 /* Don't try this at home kids... */
a5a737e0
DM		 566 rdpr %cwp, %g3
567 sub %g3, 1, %g7
5afe2738
DM		 568 wrpr %g7, %cwp
569 mov %i7, %g7
a5a737e0 570 wrpr %g3, %cwp
5afe2738 571 stx %g7, [%g1 + GR_SNAP_RPC]
93dae5b7
DM		 572 sethi %hi(trap_block), %g7
573 or %g7, %lo(trap_block), %g7
574 sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2
575 add %g7, %g2, %g7
576 ldx [%g7 + TRAP_PER_CPU_THREAD], %g3
93dae5b7
DM		 577 stx %g3, [%g1 + GR_SNAP_THREAD]
578 retry
93dae5b7 579
916ca14a
DM		 580 .globl xcall_fetch_glob_pmu
581xcall_fetch_glob_pmu:
582 sethi %hi(global_cpu_snapshot), %g1
583 or %g1, %lo(global_cpu_snapshot), %g1
584 __GET_CPUID(%g2)
585 sllx %g2, 6, %g3
586 add %g1, %g3, %g1
587 rd %pic, %g7
588 stx %g7, [%g1 + (4 * 8)]
589 rd %pcr, %g7
590 stx %g7, [%g1 + (0 * 8)]
591 retry
592
593 .globl xcall_fetch_glob_pmu_n4
594xcall_fetch_glob_pmu_n4:
595 sethi %hi(global_cpu_snapshot), %g1
596 or %g1, %lo(global_cpu_snapshot), %g1
597 __GET_CPUID(%g2)
598 sllx %g2, 6, %g3
599 add %g1, %g3, %g1
600
601 ldxa [%g0] ASI_PIC, %g7
602 stx %g7, [%g1 + (4 * 8)]
603 mov 0x08, %g3
604 ldxa [%g3] ASI_PIC, %g7
605 stx %g7, [%g1 + (5 * 8)]
606 mov 0x10, %g3
607 ldxa [%g3] ASI_PIC, %g7
608 stx %g7, [%g1 + (6 * 8)]
609 mov 0x18, %g3
610 ldxa [%g3] ASI_PIC, %g7
611 stx %g7, [%g1 + (7 * 8)]
612
613 mov %o0, %g2
614 mov %o1, %g3
615 mov %o5, %g7
616
617 mov HV_FAST_VT_GET_PERFREG, %o5
618 mov 3, %o0
619 ta HV_FAST_TRAP
620 stx %o1, [%g1 + (3 * 8)]
621 mov HV_FAST_VT_GET_PERFREG, %o5
622 mov 2, %o0
623 ta HV_FAST_TRAP
624 stx %o1, [%g1 + (2 * 8)]
625 mov HV_FAST_VT_GET_PERFREG, %o5
626 mov 1, %o0
627 ta HV_FAST_TRAP
628 stx %o1, [%g1 + (1 * 8)]
629 mov HV_FAST_VT_GET_PERFREG, %o5
630 mov 0, %o0
631 ta HV_FAST_TRAP
632 stx %o1, [%g1 + (0 * 8)]
633
634 mov %g2, %o0
635 mov %g3, %o1
636 mov %g7, %o5
637
638 retry
639
1da177e4
LT		 640#ifdef DCACHE_ALIASING_POSSIBLE
641 .align 32
642 .globl xcall_flush_dcache_page_cheetah
643xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
644 sethi %hi(PAGE_SIZE), %g3
6451: subcc %g3, (1 << 5), %g3
646 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
647 membar #Sync
648 bne,pt %icc, 1b
649 nop
650 retry
651 nop
652#endif /* DCACHE_ALIASING_POSSIBLE */
653
654 .globl xcall_flush_dcache_page_spitfire
655xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
656 %g7 == kernel page virtual address
657 %g5 == (page->mapping != NULL) */
658#ifdef DCACHE_ALIASING_POSSIBLE
659 srlx %g1, (13 - 2), %g1 ! Form tag comparitor
660 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
661 sub %g3, (1 << 5), %g3 ! D$ linesize == 32
6621: ldxa [%g3] ASI_DCACHE_TAG, %g2
663 andcc %g2, 0x3, %g0
664 be,pn %xcc, 2f
665 andn %g2, 0x3, %g2
666 cmp %g2, %g1
667
668 bne,pt %xcc, 2f
669 nop
670 stxa %g0, [%g3] ASI_DCACHE_TAG
671 membar #Sync
6722: cmp %g3, 0
673 bne,pt %xcc, 1b
674 sub %g3, (1 << 5), %g3
675
676 brz,pn %g5, 2f
677#endif /* DCACHE_ALIASING_POSSIBLE */
678 sethi %hi(PAGE_SIZE), %g3
679
6801: flush %g7
681 subcc %g3, (1 << 5), %g3
682 bne,pt %icc, 1b
683 add %g7, (1 << 5), %g7
684
6852: retry
686 nop
687 nop
688
2a3a5f5d
DM		 689 /* %g5: error
690 * %g6: tlb op
691 */
692__hypervisor_tlb_xcall_error:
693 mov %g5, %g4
694 mov %g6, %g5
695 ba,pt %xcc, etrap
696 rd %pc, %g7
697 mov %l4, %o0
698 call hypervisor_tlbop_error_xcall
699 mov %l5, %o1
7697daaa 700 ba,a,pt %xcc, rtrap
2a3a5f5d 701
52bf082f 702 .globl __hypervisor_xcall_flush_tlb_mm
2a3a5f5d 703__hypervisor_xcall_flush_tlb_mm: /* 21 insns */
52bf082f
DM		 704 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
705 mov %o0, %g2
706 mov %o1, %g3
707 mov %o2, %g4
708 mov %o3, %g1
709 mov %o5, %g7
710 clr %o0 /* ARG0: CPU lists unimplemented */
711 clr %o1 /* ARG1: CPU lists unimplemented */
712 mov %g5, %o2 /* ARG2: mmu context */
713 mov HV_MMU_ALL, %o3 /* ARG3: flags */
714 mov HV_FAST_MMU_DEMAP_CTX, %o5
715 ta HV_FAST_TRAP
2a3a5f5d
DM		 716 mov HV_FAST_MMU_DEMAP_CTX, %g6
717 brnz,pn %o0, __hypervisor_tlb_xcall_error
718 mov %o0, %g5
52bf082f
DM		 719 mov %g2, %o0
720 mov %g3, %o1
721 mov %g4, %o2
722 mov %g1, %o3
723 mov %g7, %o5
724 membar #Sync
725 retry
726
f36391d2
DM		 727 .globl __hypervisor_xcall_flush_tlb_page
728__hypervisor_xcall_flush_tlb_page: /* 17 insns */
729 /* %g5=ctx, %g1=vaddr */
52bf082f
DM		 730 mov %o0, %g2
731 mov %o1, %g3
732 mov %o2, %g4
f36391d2 733 mov %g1, %o0 /* ARG0: virtual address */
52bf082f 734 mov %g5, %o1 /* ARG1: mmu context */
2a3a5f5d
DM		 735 mov HV_MMU_ALL, %o2 /* ARG2: flags */
736 srlx %o0, PAGE_SHIFT, %o0
737 sllx %o0, PAGE_SHIFT, %o0
52bf082f 738 ta HV_MMU_UNMAP_ADDR_TRAP
2a3a5f5d
DM		 739 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
740 brnz,a,pn %o0, __hypervisor_tlb_xcall_error
741 mov %o0, %g5
52bf082f
DM		 742 mov %g2, %o0
743 mov %g3, %o1
744 mov %g4, %o2
745 membar #Sync
746 retry
747
748 .globl __hypervisor_xcall_flush_tlb_kernel_range
2a3a5f5d
DM		 749__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
750 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
52bf082f
DM		 751 sethi %hi(PAGE_SIZE - 1), %g2
752 or %g2, %lo(PAGE_SIZE - 1), %g2
753 andn %g1, %g2, %g1
754 andn %g7, %g2, %g7
755 sub %g7, %g1, %g3
756 add %g2, 1, %g2
757 sub %g3, %g2, %g3
758 mov %o0, %g2
759 mov %o1, %g4
2a3a5f5d 760 mov %o2, %g7
52bf082f
DM		 7611: add %g1, %g3, %o0 /* ARG0: virtual address */
762 mov 0, %o1 /* ARG1: mmu context */
763 mov HV_MMU_ALL, %o2 /* ARG2: flags */
764 ta HV_MMU_UNMAP_ADDR_TRAP
2a3a5f5d
DM		 765 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
766 brnz,pn %o0, __hypervisor_tlb_xcall_error
767 mov %o0, %g5
52bf082f
DM		 768 sethi %hi(PAGE_SIZE), %o2
769 brnz,pt %g3, 1b
770 sub %g3, %o2, %g3
771 mov %g2, %o0
772 mov %g4, %o1
2a3a5f5d 773 mov %g7, %o2
52bf082f
DM		 774 membar #Sync
775 retry
776
1da177e4
LT		 777 /* These just get rescheduled to PIL vectors. */
778 .globl xcall_call_function
779xcall_call_function:
780 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
781 retry
782
d172ad18
DM		 783 .globl xcall_call_function_single
784xcall_call_function_single:
785 wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
786 retry
787
1da177e4
LT		 788 .globl xcall_receive_signal
789xcall_receive_signal:
790 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
791 retry
792
793 .globl xcall_capture
794xcall_capture:
795 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint
796 retry
797
ee29074d
DM		 798 .globl xcall_new_mmu_context_version
799xcall_new_mmu_context_version:
800 wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint
801 retry
802
e2fdd7fd
DM		 803#ifdef CONFIG_KGDB
804 .globl xcall_kgdb_capture
805xcall_kgdb_capture:
42cc77c8
DM		 806 wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint
807 retry
e2fdd7fd
DM		 808#endif
809
1da177e4 810#endif /* CONFIG_SMP */
52bf082f
DM		 811
812
813 .globl hypervisor_patch_cachetlbops
814hypervisor_patch_cachetlbops:
815 save %sp, -128, %sp
816
817 sethi %hi(__flush_tlb_mm), %o0
818 or %o0, %lo(__flush_tlb_mm), %o0
819 sethi %hi(__hypervisor_flush_tlb_mm), %o1
820 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
821 call tlb_patch_one
2a3a5f5d 822 mov 10, %o2
52bf082f 823
f36391d2
DM		 824 sethi %hi(__flush_tlb_page), %o0
825 or %o0, %lo(__flush_tlb_page), %o0
826 sethi %hi(__hypervisor_flush_tlb_page), %o1
827 or %o1, %lo(__hypervisor_flush_tlb_page), %o1
828 call tlb_patch_one
829 mov 11, %o2
830
52bf082f
DM		 831 sethi %hi(__flush_tlb_pending), %o0
832 or %o0, %lo(__flush_tlb_pending), %o0
833 sethi %hi(__hypervisor_flush_tlb_pending), %o1
834 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
835 call tlb_patch_one
2a3a5f5d 836 mov 16, %o2
52bf082f
DM		 837
838 sethi %hi(__flush_tlb_kernel_range), %o0
839 or %o0, %lo(__flush_tlb_kernel_range), %o0
840 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
841 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
842 call tlb_patch_one
2a3a5f5d 843 mov 16, %o2
52bf082f
DM		 844
845#ifdef DCACHE_ALIASING_POSSIBLE
846 sethi %hi(__flush_dcache_page), %o0
847 or %o0, %lo(__flush_dcache_page), %o0
848 sethi %hi(__hypervisor_flush_dcache_page), %o1
849 or %o1, %lo(__hypervisor_flush_dcache_page), %o1
850 call tlb_patch_one
851 mov 2, %o2
852#endif /* DCACHE_ALIASING_POSSIBLE */
853
854#ifdef CONFIG_SMP
855 sethi %hi(xcall_flush_tlb_mm), %o0
856 or %o0, %lo(xcall_flush_tlb_mm), %o0
857 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
858 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
859 call tlb_patch_one
2a3a5f5d 860 mov 21, %o2
52bf082f 861
f36391d2
DM		 862 sethi %hi(xcall_flush_tlb_page), %o0
863 or %o0, %lo(xcall_flush_tlb_page), %o0
864 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1
865 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
52bf082f 866 call tlb_patch_one
f36391d2 867 mov 17, %o2
52bf082f
DM		 868
869 sethi %hi(xcall_flush_tlb_kernel_range), %o0
870 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
871 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
872 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
873 call tlb_patch_one
2a3a5f5d 874 mov 25, %o2
52bf082f
DM		 875#endif /* CONFIG_SMP */
876
877 ret
878 restore
Linux kernel - Deliverables
This page took 1.031712 seconds and 5 git commands to generate.