Commit | Line | Data |
---|---|---|
74bf4312 DM |
1 | /* tsb.S: Sparc64 TSB table handling. |
2 | * | |
3 | * Copyright (C) 2006 David S. Miller <davem@davemloft.net> | |
4 | */ | |
5 | ||
6 | #include <asm/tsb.h> | |
7 | ||
8 | .text | |
9 | .align 32 | |
10 | ||
11 | /* Invoked from TLB miss handler, we are in the | |
12 | * MMU global registers and they are setup like | |
13 | * this: | |
14 | * | |
15 | * %g1: TSB entry pointer | |
16 | * %g2: available temporary | |
17 | * %g3: FAULT_CODE_{D,I}TLB | |
18 | * %g4: available temporary | |
19 | * %g5: available temporary | |
20 | * %g6: TAG TARGET | |
21 | * %g7: physical address base of the linux page | |
22 | * tables for the current address space | |
23 | */ | |
24 | .globl tsb_miss_dtlb | |
25 | tsb_miss_dtlb: | |
26 | mov TLB_TAG_ACCESS, %g4 | |
27 | ldxa [%g4] ASI_DMMU, %g4 | |
28 | ba,pt %xcc, tsb_miss_page_table_walk | |
29 | nop | |
30 | ||
31 | .globl tsb_miss_itlb | |
32 | tsb_miss_itlb: | |
33 | mov TLB_TAG_ACCESS, %g4 | |
34 | ldxa [%g4] ASI_IMMU, %g4 | |
35 | ba,pt %xcc, tsb_miss_page_table_walk | |
36 | nop | |
37 | ||
38 | tsb_miss_page_table_walk: | |
56fb4df6 DM |
39 | /* This clobbers %g1 and %g6, preserve them... */ |
40 | mov %g1, %g5 | |
41 | mov %g6, %g2 | |
42 | ||
43 | TRAP_LOAD_PGD_PHYS | |
44 | ||
45 | mov %g2, %g6 | |
46 | mov %g5, %g1 | |
47 | ||
74bf4312 DM |
48 | USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) |
49 | ||
50 | tsb_reload: | |
51 | TSB_LOCK_TAG(%g1, %g2, %g4) | |
52 | ||
53 | /* Load and check PTE. */ | |
54 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 | |
55 | brgez,a,pn %g5, tsb_do_fault | |
517af332 | 56 | TSB_STORE(%g1, %g0) |
74bf4312 | 57 | |
09f94287 DM |
58 | /* If it is larger than the base page size, don't |
59 | * bother putting it into the TSB. | |
60 | */ | |
61 | srlx %g5, 32, %g2 | |
62 | sethi %hi(_PAGE_ALL_SZ_BITS >> 32), %g4 | |
63 | sethi %hi(_PAGE_SZBITS >> 32), %g7 | |
64 | and %g2, %g4, %g2 | |
65 | cmp %g2, %g7 | |
66 | bne,a,pn %xcc, tsb_tlb_reload | |
517af332 | 67 | TSB_STORE(%g1, %g0) |
09f94287 | 68 | |
74bf4312 DM |
69 | TSB_WRITE(%g1, %g5, %g6) |
70 | ||
71 | /* Finally, load TLB and return from trap. */ | |
72 | tsb_tlb_reload: | |
73 | cmp %g3, FAULT_CODE_DTLB | |
74 | bne,pn %xcc, tsb_itlb_load | |
75 | nop | |
76 | ||
77 | tsb_dtlb_load: | |
78 | stxa %g5, [%g0] ASI_DTLB_DATA_IN | |
79 | retry | |
80 | ||
81 | tsb_itlb_load: | |
82 | stxa %g5, [%g0] ASI_ITLB_DATA_IN | |
83 | retry | |
84 | ||
85 | /* No valid entry in the page tables, do full fault | |
86 | * processing. | |
87 | */ | |
88 | ||
89 | .globl tsb_do_fault | |
90 | tsb_do_fault: | |
91 | cmp %g3, FAULT_CODE_DTLB | |
92 | rdpr %pstate, %g5 | |
93 | bne,pn %xcc, tsb_do_itlb_fault | |
94 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate | |
95 | ||
96 | tsb_do_dtlb_fault: | |
97 | rdpr %tl, %g4 | |
98 | cmp %g4, 1 | |
99 | mov TLB_TAG_ACCESS, %g4 | |
100 | ldxa [%g4] ASI_DMMU, %g5 | |
101 | be,pt %xcc, sparc64_realfault_common | |
102 | mov FAULT_CODE_DTLB, %g4 | |
103 | ba,pt %xcc, winfix_trampoline | |
104 | nop | |
105 | ||
106 | tsb_do_itlb_fault: | |
107 | rdpr %tpc, %g5 | |
108 | ba,pt %xcc, sparc64_realfault_common | |
109 | mov FAULT_CODE_ITLB, %g4 | |
110 | ||
111 | .globl sparc64_realfault_common | |
112 | sparc64_realfault_common: | |
9bc657b2 DM |
113 | /* fault code in %g4, fault address in %g5, etrap will |
114 | * preserve these two values in %l4 and %l5 respectively | |
115 | */ | |
74bf4312 DM |
116 | ba,pt %xcc, etrap ! Save trap state |
117 | 1: rd %pc, %g7 ! ... | |
9bc657b2 DM |
118 | stb %l4, [%g6 + TI_FAULT_CODE] ! Save fault code |
119 | stx %l5, [%g6 + TI_FAULT_ADDR] ! Save fault address | |
74bf4312 DM |
120 | call do_sparc64_fault ! Call fault handler |
121 | add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg | |
122 | ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state | |
123 | nop ! Delay slot (fill me) | |
124 | ||
125 | .globl winfix_trampoline | |
126 | winfix_trampoline: | |
127 | rdpr %tpc, %g3 ! Prepare winfixup TNPC | |
128 | or %g3, 0x7c, %g3 ! Compute branch offset | |
129 | wrpr %g3, %tnpc ! Write it into TNPC | |
130 | done ! Trap return | |
131 | ||
b70c0fa1 DM |
132 | /* Insert an entry into the TSB. |
133 | * | |
517af332 | 134 | * %o0: TSB entry pointer (virt or phys address) |
b70c0fa1 DM |
135 | * %o1: tag |
136 | * %o2: pte | |
137 | */ | |
138 | .align 32 | |
517af332 DM |
139 | .globl __tsb_insert |
140 | __tsb_insert: | |
b70c0fa1 DM |
141 | rdpr %pstate, %o5 |
142 | wrpr %o5, PSTATE_IE, %pstate | |
143 | TSB_LOCK_TAG(%o0, %g2, %g3) | |
144 | TSB_WRITE(%o0, %o2, %o1) | |
145 | wrpr %o5, %pstate | |
146 | retl | |
147 | nop | |
148 | ||
517af332 DM |
149 | /* Flush the given TSB entry if it has the matching |
150 | * tag. | |
151 | * | |
152 | * %o0: TSB entry pointer (virt or phys address) | |
153 | * %o1: tag | |
154 | */ | |
155 | .align 32 | |
156 | .globl tsb_flush | |
157 | tsb_flush: | |
158 | sethi %hi(TSB_TAG_LOCK_HIGH), %g2 | |
159 | 1: TSB_LOAD_TAG(%o0, %g1) | |
160 | srlx %g1, 32, %o3 | |
161 | andcc %o3, %g2, %g0 | |
162 | bne,pn %icc, 1b | |
163 | membar #LoadLoad | |
164 | cmp %g1, %o1 | |
165 | bne,pt %xcc, 2f | |
166 | clr %o3 | |
167 | TSB_CAS_TAG(%o0, %g1, %o3) | |
168 | cmp %g1, %o3 | |
169 | bne,pn %xcc, 1b | |
170 | nop | |
171 | 2: retl | |
172 | TSB_MEMBAR | |
173 | ||
74bf4312 DM |
174 | /* Reload MMU related context switch state at |
175 | * schedule() time. | |
176 | * | |
177 | * %o0: page table physical address | |
98c5584c DM |
178 | * %o1: TSB register value |
179 | * %o2: TSB virtual address | |
180 | * %o3: TSB mapping locked PTE | |
181 | * | |
182 | * We have to run this whole thing with interrupts | |
183 | * disabled so that the current cpu doesn't change | |
184 | * due to preemption. | |
74bf4312 | 185 | */ |
56fb4df6 | 186 | .align 32 |
98c5584c DM |
187 | .globl __tsb_context_switch |
188 | __tsb_context_switch: | |
56fb4df6 DM |
189 | rdpr %pstate, %o5 |
190 | wrpr %o5, PSTATE_IE, %pstate | |
74bf4312 | 191 | |
98c5584c DM |
192 | ldub [%g6 + TI_CPU], %g1 |
193 | sethi %hi(trap_block), %g2 | |
194 | sllx %g1, TRAP_BLOCK_SZ_SHIFT, %g1 | |
195 | or %g2, %lo(trap_block), %g2 | |
196 | add %g2, %g1, %g2 | |
197 | stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] | |
74bf4312 | 198 | |
74bf4312 | 199 | mov TSB_REG, %g1 |
98c5584c | 200 | stxa %o1, [%g1] ASI_DMMU |
74bf4312 DM |
201 | membar #Sync |
202 | ||
98c5584c | 203 | stxa %o1, [%g1] ASI_IMMU |
74bf4312 DM |
204 | membar #Sync |
205 | ||
98c5584c DM |
206 | brz %o2, 9f |
207 | nop | |
74bf4312 | 208 | |
6b6d0172 DM |
209 | sethi %hi(sparc64_highest_unlocked_tlb_ent), %o4 |
210 | mov TLB_TAG_ACCESS, %g1 | |
211 | lduw [%o4 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2 | |
212 | stxa %o2, [%g1] ASI_DMMU | |
213 | membar #Sync | |
214 | sllx %g2, 3, %g2 | |
215 | stxa %o3, [%g2] ASI_DTLB_DATA_ACCESS | |
216 | membar #Sync | |
74bf4312 | 217 | 9: |
56fb4df6 | 218 | wrpr %o5, %pstate |
74bf4312 DM |
219 | |
220 | retl | |
98c5584c | 221 | nop |