Commit | Line | Data |
---|---|---|
74bf4312 DM |
1 | /* tsb.S: Sparc64 TSB table handling. |
2 | * | |
3 | * Copyright (C) 2006 David S. Miller <davem@davemloft.net> | |
4 | */ | |
5 | ||
6 | #include <asm/tsb.h> | |
618e9ed9 | 7 | #include <asm/hypervisor.h> |
74bf4312 DM |
8 | |
9 | .text | |
10 | .align 32 | |
11 | ||
12 | /* Invoked from TLB miss handler, we are in the | |
13 | * MMU global registers and they are setup like | |
14 | * this: | |
15 | * | |
16 | * %g1: TSB entry pointer | |
17 | * %g2: available temporary | |
18 | * %g3: FAULT_CODE_{D,I}TLB | |
19 | * %g4: available temporary | |
20 | * %g5: available temporary | |
21 | * %g6: TAG TARGET | |
d257d5da DM |
22 | * %g7: available temporary, will be loaded by us with |
23 | * the physical address base of the linux page | |
74bf4312 DM |
24 | * tables for the current address space |
25 | */ | |
74bf4312 DM |
26 | tsb_miss_dtlb: |
27 | mov TLB_TAG_ACCESS, %g4 | |
74bf4312 | 28 | ba,pt %xcc, tsb_miss_page_table_walk |
36a68e77 | 29 | ldxa [%g4] ASI_DMMU, %g4 |
74bf4312 | 30 | |
74bf4312 DM |
31 | tsb_miss_itlb: |
32 | mov TLB_TAG_ACCESS, %g4 | |
74bf4312 | 33 | ba,pt %xcc, tsb_miss_page_table_walk |
36a68e77 | 34 | ldxa [%g4] ASI_IMMU, %g4 |
74bf4312 | 35 | |
36a68e77 DM |
36 | /* At this point we have: |
37 | * %g4 -- missing virtual address | |
38 | * %g1 -- TSB entry address | |
8b234274 | 39 | * %g6 -- TAG TARGET (vaddr >> 22) |
d257d5da | 40 | */ |
74bf4312 | 41 | tsb_miss_page_table_walk: |
ffe483d5 | 42 | TRAP_LOAD_PGD_PHYS(%g7, %g5) |
56fb4df6 | 43 | |
36a68e77 DM |
44 | /* And now we have the PGD base physical address in %g7. */ |
45 | tsb_miss_page_table_walk_sun4v_fastpath: | |
74bf4312 DM |
46 | USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) |
47 | ||
48 | tsb_reload: | |
d257d5da | 49 | TSB_LOCK_TAG(%g1, %g2, %g7) |
74bf4312 DM |
50 | |
51 | /* Load and check PTE. */ | |
52 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 | |
8b234274 DM |
53 | mov 1, %g7 |
54 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 | |
74bf4312 | 55 | brgez,a,pn %g5, tsb_do_fault |
8b234274 | 56 | TSB_STORE(%g1, %g7) |
74bf4312 | 57 | |
09f94287 DM |
58 | /* If it is larger than the base page size, don't |
59 | * bother putting it into the TSB. | |
60 | */ | |
c4bce90e DM |
61 | sethi %hi(_PAGE_ALL_SZ_BITS), %g7 |
62 | ldx [%g7 + %lo(_PAGE_ALL_SZ_BITS)], %g7 | |
63 | and %g5, %g7, %g2 | |
64 | sethi %hi(_PAGE_SZBITS), %g7 | |
65 | ldx [%g7 + %lo(_PAGE_SZBITS)], %g7 | |
09f94287 | 66 | cmp %g2, %g7 |
8b234274 DM |
67 | mov 1, %g7 |
68 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 | |
09f94287 | 69 | bne,a,pn %xcc, tsb_tlb_reload |
8b234274 | 70 | TSB_STORE(%g1, %g7) |
09f94287 | 71 | |
74bf4312 DM |
72 | TSB_WRITE(%g1, %g5, %g6) |
73 | ||
74 | /* Finally, load TLB and return from trap. */ | |
75 | tsb_tlb_reload: | |
76 | cmp %g3, FAULT_CODE_DTLB | |
77 | bne,pn %xcc, tsb_itlb_load | |
78 | nop | |
79 | ||
80 | tsb_dtlb_load: | |
d257d5da DM |
81 | |
82 | 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN | |
74bf4312 | 83 | retry |
df7d6aec | 84 | .section .sun4v_2insn_patch, "ax" |
d257d5da DM |
85 | .word 661b |
86 | nop | |
87 | nop | |
88 | .previous | |
89 | ||
90 | /* For sun4v the ASI_DTLB_DATA_IN store and the retry | |
91 | * instruction get nop'd out and we get here to branch | |
92 | * to the sun4v tlb load code. The registers are setup | |
93 | * as follows: | |
94 | * | |
95 | * %g4: vaddr | |
96 | * %g5: PTE | |
97 | * %g6: TAG | |
98 | * | |
99 | * The sun4v TLB load wants the PTE in %g3 so we fix that | |
100 | * up here. | |
101 | */ | |
102 | ba,pt %xcc, sun4v_dtlb_load | |
103 | mov %g5, %g3 | |
74bf4312 DM |
104 | |
105 | tsb_itlb_load: | |
d257d5da DM |
106 | |
107 | 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN | |
74bf4312 | 108 | retry |
df7d6aec | 109 | .section .sun4v_2insn_patch, "ax" |
d257d5da DM |
110 | .word 661b |
111 | nop | |
112 | nop | |
113 | .previous | |
114 | ||
115 | /* For sun4v the ASI_ITLB_DATA_IN store and the retry | |
116 | * instruction get nop'd out and we get here to branch | |
117 | * to the sun4v tlb load code. The registers are setup | |
118 | * as follows: | |
119 | * | |
120 | * %g4: vaddr | |
121 | * %g5: PTE | |
122 | * %g6: TAG | |
123 | * | |
124 | * The sun4v TLB load wants the PTE in %g3 so we fix that | |
125 | * up here. | |
126 | */ | |
127 | ba,pt %xcc, sun4v_itlb_load | |
128 | mov %g5, %g3 | |
74bf4312 DM |
129 | |
130 | /* No valid entry in the page tables, do full fault | |
131 | * processing. | |
132 | */ | |
133 | ||
134 | .globl tsb_do_fault | |
135 | tsb_do_fault: | |
136 | cmp %g3, FAULT_CODE_DTLB | |
45fec05f DM |
137 | |
138 | 661: rdpr %pstate, %g5 | |
139 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate | |
df7d6aec | 140 | .section .sun4v_2insn_patch, "ax" |
45fec05f | 141 | .word 661b |
6c8927c9 | 142 | SET_GL(1) |
8b234274 | 143 | ldxa [%g0] ASI_SCRATCHPAD, %g4 |
45fec05f DM |
144 | .previous |
145 | ||
74bf4312 | 146 | bne,pn %xcc, tsb_do_itlb_fault |
45fec05f | 147 | nop |
74bf4312 DM |
148 | |
149 | tsb_do_dtlb_fault: | |
d257d5da DM |
150 | rdpr %tl, %g3 |
151 | cmp %g3, 1 | |
152 | ||
153 | 661: mov TLB_TAG_ACCESS, %g4 | |
74bf4312 | 154 | ldxa [%g4] ASI_DMMU, %g5 |
df7d6aec | 155 | .section .sun4v_2insn_patch, "ax" |
d257d5da | 156 | .word 661b |
8b234274 | 157 | ldx [%g4 + HV_FAULT_D_ADDR_OFFSET], %g5 |
d257d5da DM |
158 | nop |
159 | .previous | |
160 | ||
74bf4312 DM |
161 | be,pt %xcc, sparc64_realfault_common |
162 | mov FAULT_CODE_DTLB, %g4 | |
163 | ba,pt %xcc, winfix_trampoline | |
164 | nop | |
165 | ||
166 | tsb_do_itlb_fault: | |
167 | rdpr %tpc, %g5 | |
168 | ba,pt %xcc, sparc64_realfault_common | |
169 | mov FAULT_CODE_ITLB, %g4 | |
170 | ||
171 | .globl sparc64_realfault_common | |
172 | sparc64_realfault_common: | |
9bc657b2 DM |
173 | /* fault code in %g4, fault address in %g5, etrap will |
174 | * preserve these two values in %l4 and %l5 respectively | |
175 | */ | |
74bf4312 DM |
176 | ba,pt %xcc, etrap ! Save trap state |
177 | 1: rd %pc, %g7 ! ... | |
9bc657b2 DM |
178 | stb %l4, [%g6 + TI_FAULT_CODE] ! Save fault code |
179 | stx %l5, [%g6 + TI_FAULT_ADDR] ! Save fault address | |
74bf4312 DM |
180 | call do_sparc64_fault ! Call fault handler |
181 | add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg | |
182 | ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state | |
183 | nop ! Delay slot (fill me) | |
184 | ||
74bf4312 DM |
185 | winfix_trampoline: |
186 | rdpr %tpc, %g3 ! Prepare winfixup TNPC | |
187 | or %g3, 0x7c, %g3 ! Compute branch offset | |
188 | wrpr %g3, %tnpc ! Write it into TNPC | |
189 | done ! Trap return | |
190 | ||
b70c0fa1 DM |
191 | /* Insert an entry into the TSB. |
192 | * | |
517af332 | 193 | * %o0: TSB entry pointer (virt or phys address) |
b70c0fa1 DM |
194 | * %o1: tag |
195 | * %o2: pte | |
196 | */ | |
197 | .align 32 | |
517af332 DM |
198 | .globl __tsb_insert |
199 | __tsb_insert: | |
b70c0fa1 DM |
200 | rdpr %pstate, %o5 |
201 | wrpr %o5, PSTATE_IE, %pstate | |
202 | TSB_LOCK_TAG(%o0, %g2, %g3) | |
203 | TSB_WRITE(%o0, %o2, %o1) | |
204 | wrpr %o5, %pstate | |
205 | retl | |
206 | nop | |
207 | ||
517af332 DM |
208 | /* Flush the given TSB entry if it has the matching |
209 | * tag. | |
210 | * | |
211 | * %o0: TSB entry pointer (virt or phys address) | |
212 | * %o1: tag | |
213 | */ | |
214 | .align 32 | |
215 | .globl tsb_flush | |
216 | tsb_flush: | |
217 | sethi %hi(TSB_TAG_LOCK_HIGH), %g2 | |
218 | 1: TSB_LOAD_TAG(%o0, %g1) | |
219 | srlx %g1, 32, %o3 | |
220 | andcc %o3, %g2, %g0 | |
221 | bne,pn %icc, 1b | |
222 | membar #LoadLoad | |
223 | cmp %g1, %o1 | |
8b234274 | 224 | mov 1, %o3 |
517af332 | 225 | bne,pt %xcc, 2f |
8b234274 | 226 | sllx %o3, TSB_TAG_INVALID_BIT, %o3 |
517af332 DM |
227 | TSB_CAS_TAG(%o0, %g1, %o3) |
228 | cmp %g1, %o3 | |
229 | bne,pn %xcc, 1b | |
230 | nop | |
231 | 2: retl | |
232 | TSB_MEMBAR | |
233 | ||
74bf4312 DM |
234 | /* Reload MMU related context switch state at |
235 | * schedule() time. | |
236 | * | |
237 | * %o0: page table physical address | |
98c5584c DM |
238 | * %o1: TSB register value |
239 | * %o2: TSB virtual address | |
240 | * %o3: TSB mapping locked PTE | |
618e9ed9 | 241 | * %o4: Hypervisor TSB descriptor physical address |
98c5584c DM |
242 | * |
243 | * We have to run this whole thing with interrupts | |
244 | * disabled so that the current cpu doesn't change | |
245 | * due to preemption. | |
74bf4312 | 246 | */ |
56fb4df6 | 247 | .align 32 |
98c5584c DM |
248 | .globl __tsb_context_switch |
249 | __tsb_context_switch: | |
56fb4df6 DM |
250 | rdpr %pstate, %o5 |
251 | wrpr %o5, PSTATE_IE, %pstate | |
74bf4312 | 252 | |
98c5584c DM |
253 | ldub [%g6 + TI_CPU], %g1 |
254 | sethi %hi(trap_block), %g2 | |
255 | sllx %g1, TRAP_BLOCK_SZ_SHIFT, %g1 | |
256 | or %g2, %lo(trap_block), %g2 | |
257 | add %g2, %g1, %g2 | |
258 | stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] | |
74bf4312 | 259 | |
618e9ed9 DM |
260 | sethi %hi(tlb_type), %g1 |
261 | lduw [%g1 + %lo(tlb_type)], %g1 | |
262 | cmp %g1, 3 | |
263 | bne,pt %icc, 1f | |
264 | nop | |
265 | ||
266 | /* Hypervisor TSB switch. */ | |
d257d5da DM |
267 | mov SCRATCHPAD_UTSBREG1, %g1 |
268 | stxa %o1, [%g1] ASI_SCRATCHPAD | |
618e9ed9 DM |
269 | mov -1, %g2 |
270 | mov SCRATCHPAD_UTSBREG2, %g1 | |
271 | stxa %g2, [%g1] ASI_SCRATCHPAD | |
d257d5da | 272 | |
a7b31bac DM |
273 | /* Save away %o5's %pstate, we have to use %o5 for |
274 | * the hypervisor call. | |
275 | */ | |
276 | mov %o5, %g1 | |
277 | ||
164c220f DM |
278 | mov HV_FAST_MMU_TSB_CTXNON0, %o5 |
279 | mov 1, %o0 | |
280 | mov %o4, %o1 | |
618e9ed9 DM |
281 | ta HV_FAST_TRAP |
282 | ||
a7b31bac | 283 | /* Finish up and restore %o5. */ |
618e9ed9 | 284 | ba,pt %xcc, 9f |
a7b31bac | 285 | mov %g1, %o5 |
74bf4312 | 286 | |
618e9ed9 DM |
287 | /* SUN4U TSB switch. */ |
288 | 1: mov TSB_REG, %g1 | |
289 | stxa %o1, [%g1] ASI_DMMU | |
290 | membar #Sync | |
291 | stxa %o1, [%g1] ASI_IMMU | |
74bf4312 DM |
292 | membar #Sync |
293 | ||
618e9ed9 | 294 | 2: brz %o2, 9f |
98c5584c | 295 | nop |
74bf4312 | 296 | |
618e9ed9 | 297 | sethi %hi(sparc64_highest_unlocked_tlb_ent), %g2 |
6b6d0172 | 298 | mov TLB_TAG_ACCESS, %g1 |
618e9ed9 | 299 | lduw [%g2 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2 |
6b6d0172 DM |
300 | stxa %o2, [%g1] ASI_DMMU |
301 | membar #Sync | |
302 | sllx %g2, 3, %g2 | |
303 | stxa %o3, [%g2] ASI_DTLB_DATA_ACCESS | |
304 | membar #Sync | |
74bf4312 | 305 | 9: |
56fb4df6 | 306 | wrpr %o5, %pstate |
74bf4312 DM |
307 | |
308 | retl | |
98c5584c | 309 | nop |