Commit | Line | Data |
---|---|---|
74bf4312 DM |
1 | /* tsb.S: Sparc64 TSB table handling. |
2 | * | |
3 | * Copyright (C) 2006 David S. Miller <davem@davemloft.net> | |
4 | */ | |
5 | ||
6 | #include <asm/tsb.h> | |
7 | ||
8 | .text | |
9 | .align 32 | |
10 | ||
11 | /* Invoked from TLB miss handler, we are in the | |
12 | * MMU global registers and they are setup like | |
13 | * this: | |
14 | * | |
15 | * %g1: TSB entry pointer | |
16 | * %g2: available temporary | |
17 | * %g3: FAULT_CODE_{D,I}TLB | |
18 | * %g4: available temporary | |
19 | * %g5: available temporary | |
20 | * %g6: TAG TARGET | |
d257d5da DM |
21 | * %g7: available temporary, will be loaded by us with |
22 | * the physical address base of the linux page | |
74bf4312 DM |
23 | * tables for the current address space |
24 | */ | |
74bf4312 DM |
25 | tsb_miss_dtlb: |
26 | mov TLB_TAG_ACCESS, %g4 | |
27 | ldxa [%g4] ASI_DMMU, %g4 | |
28 | ba,pt %xcc, tsb_miss_page_table_walk | |
29 | nop | |
30 | ||
74bf4312 DM |
31 | tsb_miss_itlb: |
32 | mov TLB_TAG_ACCESS, %g4 | |
33 | ldxa [%g4] ASI_IMMU, %g4 | |
34 | ba,pt %xcc, tsb_miss_page_table_walk | |
35 | nop | |
36 | ||
d257d5da DM |
37 | /* The sun4v TLB miss handlers jump directly here instead |
38 | * of tsb_miss_{d,i}tlb with the missing virtual address | |
39 | * already loaded into %g4. | |
40 | */ | |
74bf4312 | 41 | tsb_miss_page_table_walk: |
ffe483d5 | 42 | TRAP_LOAD_PGD_PHYS(%g7, %g5) |
56fb4df6 | 43 | |
74bf4312 DM |
44 | USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) |
45 | ||
46 | tsb_reload: | |
d257d5da | 47 | TSB_LOCK_TAG(%g1, %g2, %g7) |
74bf4312 DM |
48 | |
49 | /* Load and check PTE. */ | |
50 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 | |
51 | brgez,a,pn %g5, tsb_do_fault | |
517af332 | 52 | TSB_STORE(%g1, %g0) |
74bf4312 | 53 | |
09f94287 DM |
54 | /* If it is larger than the base page size, don't |
55 | * bother putting it into the TSB. | |
56 | */ | |
57 | srlx %g5, 32, %g2 | |
d257d5da DM |
58 | sethi %hi(_PAGE_ALL_SZ_BITS >> 32), %g7 |
59 | and %g2, %g7, %g2 | |
09f94287 | 60 | sethi %hi(_PAGE_SZBITS >> 32), %g7 |
09f94287 DM |
61 | cmp %g2, %g7 |
62 | bne,a,pn %xcc, tsb_tlb_reload | |
517af332 | 63 | TSB_STORE(%g1, %g0) |
09f94287 | 64 | |
74bf4312 DM |
65 | TSB_WRITE(%g1, %g5, %g6) |
66 | ||
67 | /* Finally, load TLB and return from trap. */ | |
68 | tsb_tlb_reload: | |
69 | cmp %g3, FAULT_CODE_DTLB | |
70 | bne,pn %xcc, tsb_itlb_load | |
71 | nop | |
72 | ||
73 | tsb_dtlb_load: | |
d257d5da DM |
74 | |
75 | 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN | |
74bf4312 | 76 | retry |
df7d6aec | 77 | .section .sun4v_2insn_patch, "ax" |
d257d5da DM |
78 | .word 661b |
79 | nop | |
80 | nop | |
81 | .previous | |
82 | ||
83 | /* For sun4v the ASI_DTLB_DATA_IN store and the retry | |
84 | * instruction get nop'd out and we get here to branch | |
85 | * to the sun4v tlb load code. The registers are setup | |
86 | * as follows: | |
87 | * | |
88 | * %g4: vaddr | |
89 | * %g5: PTE | |
90 | * %g6: TAG | |
91 | * | |
92 | * The sun4v TLB load wants the PTE in %g3 so we fix that | |
93 | * up here. | |
94 | */ | |
95 | ba,pt %xcc, sun4v_dtlb_load | |
96 | mov %g5, %g3 | |
74bf4312 DM |
97 | |
98 | tsb_itlb_load: | |
d257d5da DM |
99 | |
100 | 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN | |
74bf4312 | 101 | retry |
df7d6aec | 102 | .section .sun4v_2insn_patch, "ax" |
d257d5da DM |
103 | .word 661b |
104 | nop | |
105 | nop | |
106 | .previous | |
107 | ||
108 | /* For sun4v the ASI_ITLB_DATA_IN store and the retry | |
109 | * instruction get nop'd out and we get here to branch | |
110 | * to the sun4v tlb load code. The registers are setup | |
111 | * as follows: | |
112 | * | |
113 | * %g4: vaddr | |
114 | * %g5: PTE | |
115 | * %g6: TAG | |
116 | * | |
117 | * The sun4v TLB load wants the PTE in %g3 so we fix that | |
118 | * up here. | |
119 | */ | |
120 | ba,pt %xcc, sun4v_itlb_load | |
121 | mov %g5, %g3 | |
74bf4312 DM |
122 | |
123 | /* No valid entry in the page tables, do full fault | |
124 | * processing. | |
125 | */ | |
126 | ||
127 | .globl tsb_do_fault | |
128 | tsb_do_fault: | |
129 | cmp %g3, FAULT_CODE_DTLB | |
45fec05f DM |
130 | |
131 | 661: rdpr %pstate, %g5 | |
132 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate | |
df7d6aec | 133 | .section .sun4v_2insn_patch, "ax" |
45fec05f DM |
134 | .word 661b |
135 | nop | |
136 | nop | |
137 | .previous | |
138 | ||
74bf4312 | 139 | bne,pn %xcc, tsb_do_itlb_fault |
45fec05f | 140 | nop |
74bf4312 DM |
141 | |
142 | tsb_do_dtlb_fault: | |
d257d5da DM |
143 | rdpr %tl, %g3 |
144 | cmp %g3, 1 | |
145 | ||
146 | 661: mov TLB_TAG_ACCESS, %g4 | |
74bf4312 | 147 | ldxa [%g4] ASI_DMMU, %g5 |
df7d6aec | 148 | .section .sun4v_2insn_patch, "ax" |
d257d5da DM |
149 | .word 661b |
150 | mov %g4, %g5 | |
151 | nop | |
152 | .previous | |
153 | ||
74bf4312 DM |
154 | be,pt %xcc, sparc64_realfault_common |
155 | mov FAULT_CODE_DTLB, %g4 | |
156 | ba,pt %xcc, winfix_trampoline | |
157 | nop | |
158 | ||
159 | tsb_do_itlb_fault: | |
160 | rdpr %tpc, %g5 | |
161 | ba,pt %xcc, sparc64_realfault_common | |
162 | mov FAULT_CODE_ITLB, %g4 | |
163 | ||
164 | .globl sparc64_realfault_common | |
165 | sparc64_realfault_common: | |
9bc657b2 DM |
166 | /* fault code in %g4, fault address in %g5, etrap will |
167 | * preserve these two values in %l4 and %l5 respectively | |
168 | */ | |
74bf4312 DM |
169 | ba,pt %xcc, etrap ! Save trap state |
170 | 1: rd %pc, %g7 ! ... | |
9bc657b2 DM |
171 | stb %l4, [%g6 + TI_FAULT_CODE] ! Save fault code |
172 | stx %l5, [%g6 + TI_FAULT_ADDR] ! Save fault address | |
74bf4312 DM |
173 | call do_sparc64_fault ! Call fault handler |
174 | add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg | |
175 | ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state | |
176 | nop ! Delay slot (fill me) | |
177 | ||
74bf4312 DM |
178 | winfix_trampoline: |
179 | rdpr %tpc, %g3 ! Prepare winfixup TNPC | |
180 | or %g3, 0x7c, %g3 ! Compute branch offset | |
181 | wrpr %g3, %tnpc ! Write it into TNPC | |
182 | done ! Trap return | |
183 | ||
b70c0fa1 DM |
184 | /* Insert an entry into the TSB. |
185 | * | |
517af332 | 186 | * %o0: TSB entry pointer (virt or phys address) |
b70c0fa1 DM |
187 | * %o1: tag |
188 | * %o2: pte | |
189 | */ | |
190 | .align 32 | |
517af332 DM |
191 | .globl __tsb_insert |
192 | __tsb_insert: | |
b70c0fa1 DM |
193 | rdpr %pstate, %o5 |
194 | wrpr %o5, PSTATE_IE, %pstate | |
195 | TSB_LOCK_TAG(%o0, %g2, %g3) | |
196 | TSB_WRITE(%o0, %o2, %o1) | |
197 | wrpr %o5, %pstate | |
198 | retl | |
199 | nop | |
200 | ||
517af332 DM |
201 | /* Flush the given TSB entry if it has the matching |
202 | * tag. | |
203 | * | |
204 | * %o0: TSB entry pointer (virt or phys address) | |
205 | * %o1: tag | |
206 | */ | |
207 | .align 32 | |
208 | .globl tsb_flush | |
209 | tsb_flush: | |
210 | sethi %hi(TSB_TAG_LOCK_HIGH), %g2 | |
211 | 1: TSB_LOAD_TAG(%o0, %g1) | |
212 | srlx %g1, 32, %o3 | |
213 | andcc %o3, %g2, %g0 | |
214 | bne,pn %icc, 1b | |
215 | membar #LoadLoad | |
216 | cmp %g1, %o1 | |
217 | bne,pt %xcc, 2f | |
218 | clr %o3 | |
219 | TSB_CAS_TAG(%o0, %g1, %o3) | |
220 | cmp %g1, %o3 | |
221 | bne,pn %xcc, 1b | |
222 | nop | |
223 | 2: retl | |
224 | TSB_MEMBAR | |
225 | ||
74bf4312 DM |
226 | /* Reload MMU related context switch state at |
227 | * schedule() time. | |
228 | * | |
229 | * %o0: page table physical address | |
98c5584c DM |
230 | * %o1: TSB register value |
231 | * %o2: TSB virtual address | |
232 | * %o3: TSB mapping locked PTE | |
233 | * | |
234 | * We have to run this whole thing with interrupts | |
235 | * disabled so that the current cpu doesn't change | |
236 | * due to preemption. | |
74bf4312 | 237 | */ |
56fb4df6 | 238 | .align 32 |
98c5584c DM |
239 | .globl __tsb_context_switch |
240 | __tsb_context_switch: | |
56fb4df6 DM |
241 | rdpr %pstate, %o5 |
242 | wrpr %o5, PSTATE_IE, %pstate | |
74bf4312 | 243 | |
98c5584c DM |
244 | ldub [%g6 + TI_CPU], %g1 |
245 | sethi %hi(trap_block), %g2 | |
246 | sllx %g1, TRAP_BLOCK_SZ_SHIFT, %g1 | |
247 | or %g2, %lo(trap_block), %g2 | |
248 | add %g2, %g1, %g2 | |
249 | stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] | |
74bf4312 | 250 | |
d257d5da | 251 | 661: mov TSB_REG, %g1 |
98c5584c | 252 | stxa %o1, [%g1] ASI_DMMU |
df7d6aec | 253 | .section .sun4v_2insn_patch, "ax" |
d257d5da DM |
254 | .word 661b |
255 | mov SCRATCHPAD_UTSBREG1, %g1 | |
256 | stxa %o1, [%g1] ASI_SCRATCHPAD | |
257 | .previous | |
258 | ||
74bf4312 DM |
259 | membar #Sync |
260 | ||
d257d5da | 261 | 661: stxa %o1, [%g1] ASI_IMMU |
74bf4312 | 262 | membar #Sync |
df7d6aec | 263 | .section .sun4v_2insn_patch, "ax" |
d257d5da DM |
264 | .word 661b |
265 | nop | |
266 | nop | |
267 | .previous | |
74bf4312 | 268 | |
98c5584c DM |
269 | brz %o2, 9f |
270 | nop | |
74bf4312 | 271 | |
6b6d0172 DM |
272 | sethi %hi(sparc64_highest_unlocked_tlb_ent), %o4 |
273 | mov TLB_TAG_ACCESS, %g1 | |
274 | lduw [%o4 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2 | |
275 | stxa %o2, [%g1] ASI_DMMU | |
276 | membar #Sync | |
277 | sllx %g2, 3, %g2 | |
278 | stxa %o3, [%g2] ASI_DTLB_DATA_ACCESS | |
279 | membar #Sync | |
74bf4312 | 280 | 9: |
56fb4df6 | 281 | wrpr %o5, %pstate |
74bf4312 DM |
282 | |
283 | retl | |
98c5584c | 284 | nop |