Commit | Line | Data |
---|---|---|
88278ca2 | 1 | /* |
1da177e4 LT |
2 | * hypersparc.S: High speed Hypersparc mmu/cache operations. |
3 | * | |
4 | * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) | |
5 | */ | |
6 | ||
7 | #include <asm/ptrace.h> | |
8 | #include <asm/psr.h> | |
47003497 | 9 | #include <asm/asm-offsets.h> |
1da177e4 LT |
10 | #include <asm/asi.h> |
11 | #include <asm/page.h> | |
12 | #include <asm/pgtsrmmu.h> | |
1da177e4 LT |
13 | #include <linux/init.h> |
14 | ||
15 | .text | |
16 | .align 4 | |
17 | ||
18 | .globl hypersparc_flush_cache_all, hypersparc_flush_cache_mm | |
19 | .globl hypersparc_flush_cache_range, hypersparc_flush_cache_page | |
20 | .globl hypersparc_flush_page_to_ram | |
21 | .globl hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns | |
22 | .globl hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm | |
23 | .globl hypersparc_flush_tlb_range, hypersparc_flush_tlb_page | |
24 | ||
25 | hypersparc_flush_cache_all: | |
26 | WINDOW_FLUSH(%g4, %g5) | |
27 | sethi %hi(vac_cache_size), %g4 | |
28 | ld [%g4 + %lo(vac_cache_size)], %g5 | |
29 | sethi %hi(vac_line_size), %g1 | |
30 | ld [%g1 + %lo(vac_line_size)], %g2 | |
31 | 1: | |
32 | subcc %g5, %g2, %g5 ! hyper_flush_unconditional_combined | |
33 | bne 1b | |
34 | sta %g0, [%g5] ASI_M_FLUSH_CTX | |
35 | retl | |
36 | sta %g0, [%g0] ASI_M_FLUSH_IWHOLE ! hyper_flush_whole_icache | |
37 | ||
38 | /* We expand the window flush to get maximum performance. */ | |
39 | hypersparc_flush_cache_mm: | |
40 | #ifndef CONFIG_SMP | |
41 | ld [%o0 + AOFF_mm_context], %g1 | |
42 | cmp %g1, -1 | |
43 | be hypersparc_flush_cache_mm_out | |
44 | #endif | |
45 | WINDOW_FLUSH(%g4, %g5) | |
46 | ||
47 | sethi %hi(vac_line_size), %g1 | |
48 | ld [%g1 + %lo(vac_line_size)], %o1 | |
49 | sethi %hi(vac_cache_size), %g2 | |
50 | ld [%g2 + %lo(vac_cache_size)], %o0 | |
51 | add %o1, %o1, %g1 | |
52 | add %o1, %g1, %g2 | |
53 | add %o1, %g2, %g3 | |
54 | add %o1, %g3, %g4 | |
55 | add %o1, %g4, %g5 | |
56 | add %o1, %g5, %o4 | |
57 | add %o1, %o4, %o5 | |
58 | ||
59 | /* BLAMMO! */ | |
60 | 1: | |
61 | subcc %o0, %o5, %o0 ! hyper_flush_cache_user | |
62 | sta %g0, [%o0 + %g0] ASI_M_FLUSH_USER | |
63 | sta %g0, [%o0 + %o1] ASI_M_FLUSH_USER | |
64 | sta %g0, [%o0 + %g1] ASI_M_FLUSH_USER | |
65 | sta %g0, [%o0 + %g2] ASI_M_FLUSH_USER | |
66 | sta %g0, [%o0 + %g3] ASI_M_FLUSH_USER | |
67 | sta %g0, [%o0 + %g4] ASI_M_FLUSH_USER | |
68 | sta %g0, [%o0 + %g5] ASI_M_FLUSH_USER | |
69 | bne 1b | |
70 | sta %g0, [%o0 + %o4] ASI_M_FLUSH_USER | |
71 | hypersparc_flush_cache_mm_out: | |
72 | retl | |
73 | nop | |
74 | ||
75 | /* The things we do for performance... */ | |
76 | hypersparc_flush_cache_range: | |
961246b4 | 77 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
78 | #ifndef CONFIG_SMP |
79 | ld [%o0 + AOFF_mm_context], %g1 | |
80 | cmp %g1, -1 | |
81 | be hypersparc_flush_cache_range_out | |
82 | #endif | |
83 | WINDOW_FLUSH(%g4, %g5) | |
84 | ||
85 | sethi %hi(vac_line_size), %g1 | |
86 | ld [%g1 + %lo(vac_line_size)], %o4 | |
87 | sethi %hi(vac_cache_size), %g2 | |
88 | ld [%g2 + %lo(vac_cache_size)], %o3 | |
89 | ||
90 | /* Here comes the fun part... */ | |
91 | add %o2, (PAGE_SIZE - 1), %o2 | |
92 | andn %o1, (PAGE_SIZE - 1), %o1 | |
93 | add %o4, %o4, %o5 | |
94 | andn %o2, (PAGE_SIZE - 1), %o2 | |
95 | add %o4, %o5, %g1 | |
96 | sub %o2, %o1, %g4 | |
97 | add %o4, %g1, %g2 | |
98 | sll %o3, 2, %g5 | |
99 | add %o4, %g2, %g3 | |
100 | cmp %g4, %g5 | |
101 | add %o4, %g3, %g4 | |
102 | blu 0f | |
103 | add %o4, %g4, %g5 | |
104 | add %o4, %g5, %g7 | |
105 | ||
106 | /* Flush entire user space, believe it or not this is quicker | |
107 | * than page at a time flushings for range > (cache_size<<2). | |
108 | */ | |
109 | 1: | |
110 | subcc %o3, %g7, %o3 | |
111 | sta %g0, [%o3 + %g0] ASI_M_FLUSH_USER | |
112 | sta %g0, [%o3 + %o4] ASI_M_FLUSH_USER | |
113 | sta %g0, [%o3 + %o5] ASI_M_FLUSH_USER | |
114 | sta %g0, [%o3 + %g1] ASI_M_FLUSH_USER | |
115 | sta %g0, [%o3 + %g2] ASI_M_FLUSH_USER | |
116 | sta %g0, [%o3 + %g3] ASI_M_FLUSH_USER | |
117 | sta %g0, [%o3 + %g4] ASI_M_FLUSH_USER | |
118 | bne 1b | |
119 | sta %g0, [%o3 + %g5] ASI_M_FLUSH_USER | |
120 | retl | |
121 | nop | |
122 | ||
123 | /* Below our threshold, flush one page at a time. */ | |
124 | 0: | |
125 | ld [%o0 + AOFF_mm_context], %o0 | |
126 | mov SRMMU_CTX_REG, %g7 | |
127 | lda [%g7] ASI_M_MMUREGS, %o3 | |
128 | sta %o0, [%g7] ASI_M_MMUREGS | |
129 | add %o2, -PAGE_SIZE, %o0 | |
130 | 1: | |
131 | or %o0, 0x400, %g7 | |
132 | lda [%g7] ASI_M_FLUSH_PROBE, %g7 | |
133 | orcc %g7, 0, %g0 | |
134 | be,a 3f | |
135 | mov %o0, %o2 | |
136 | add %o4, %g5, %g7 | |
137 | 2: | |
138 | sub %o2, %g7, %o2 | |
139 | sta %g0, [%o2 + %g0] ASI_M_FLUSH_PAGE | |
140 | sta %g0, [%o2 + %o4] ASI_M_FLUSH_PAGE | |
141 | sta %g0, [%o2 + %o5] ASI_M_FLUSH_PAGE | |
142 | sta %g0, [%o2 + %g1] ASI_M_FLUSH_PAGE | |
143 | sta %g0, [%o2 + %g2] ASI_M_FLUSH_PAGE | |
144 | sta %g0, [%o2 + %g3] ASI_M_FLUSH_PAGE | |
145 | andcc %o2, 0xffc, %g0 | |
146 | sta %g0, [%o2 + %g4] ASI_M_FLUSH_PAGE | |
147 | bne 2b | |
148 | sta %g0, [%o2 + %g5] ASI_M_FLUSH_PAGE | |
149 | 3: | |
150 | cmp %o2, %o1 | |
151 | bne 1b | |
152 | add %o2, -PAGE_SIZE, %o0 | |
153 | mov SRMMU_FAULT_STATUS, %g5 | |
154 | lda [%g5] ASI_M_MMUREGS, %g0 | |
155 | mov SRMMU_CTX_REG, %g7 | |
156 | sta %o3, [%g7] ASI_M_MMUREGS | |
157 | hypersparc_flush_cache_range_out: | |
158 | retl | |
159 | nop | |
160 | ||
161 | /* HyperSparc requires a valid mapping where we are about to flush | |
162 | * in order to check for a physical tag match during the flush. | |
163 | */ | |
164 | /* Verified, my ass... */ | |
165 | hypersparc_flush_cache_page: | |
961246b4 | 166 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
167 | ld [%o0 + AOFF_mm_context], %g2 |
168 | #ifndef CONFIG_SMP | |
169 | cmp %g2, -1 | |
170 | be hypersparc_flush_cache_page_out | |
171 | #endif | |
172 | WINDOW_FLUSH(%g4, %g5) | |
173 | ||
174 | sethi %hi(vac_line_size), %g1 | |
175 | ld [%g1 + %lo(vac_line_size)], %o4 | |
176 | mov SRMMU_CTX_REG, %o3 | |
177 | andn %o1, (PAGE_SIZE - 1), %o1 | |
178 | lda [%o3] ASI_M_MMUREGS, %o2 | |
179 | sta %g2, [%o3] ASI_M_MMUREGS | |
180 | or %o1, 0x400, %o5 | |
181 | lda [%o5] ASI_M_FLUSH_PROBE, %g1 | |
182 | orcc %g0, %g1, %g0 | |
183 | be 2f | |
184 | add %o4, %o4, %o5 | |
185 | sub %o1, -PAGE_SIZE, %o1 | |
186 | add %o4, %o5, %g1 | |
187 | add %o4, %g1, %g2 | |
188 | add %o4, %g2, %g3 | |
189 | add %o4, %g3, %g4 | |
190 | add %o4, %g4, %g5 | |
191 | add %o4, %g5, %g7 | |
192 | ||
193 | /* BLAMMO! */ | |
194 | 1: | |
195 | sub %o1, %g7, %o1 | |
196 | sta %g0, [%o1 + %g0] ASI_M_FLUSH_PAGE | |
197 | sta %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE | |
198 | sta %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE | |
199 | sta %g0, [%o1 + %g1] ASI_M_FLUSH_PAGE | |
200 | sta %g0, [%o1 + %g2] ASI_M_FLUSH_PAGE | |
201 | sta %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE | |
202 | andcc %o1, 0xffc, %g0 | |
203 | sta %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE | |
204 | bne 1b | |
205 | sta %g0, [%o1 + %g5] ASI_M_FLUSH_PAGE | |
206 | 2: | |
207 | mov SRMMU_FAULT_STATUS, %g7 | |
208 | mov SRMMU_CTX_REG, %g4 | |
209 | lda [%g7] ASI_M_MMUREGS, %g0 | |
210 | sta %o2, [%g4] ASI_M_MMUREGS | |
211 | hypersparc_flush_cache_page_out: | |
212 | retl | |
213 | nop | |
214 | ||
215 | hypersparc_flush_sig_insns: | |
216 | flush %o1 | |
217 | retl | |
218 | flush %o1 + 4 | |
219 | ||
220 | /* HyperSparc is copy-back. */ | |
221 | hypersparc_flush_page_to_ram: | |
222 | sethi %hi(vac_line_size), %g1 | |
223 | ld [%g1 + %lo(vac_line_size)], %o4 | |
224 | andn %o0, (PAGE_SIZE - 1), %o0 | |
225 | add %o4, %o4, %o5 | |
226 | or %o0, 0x400, %g7 | |
227 | lda [%g7] ASI_M_FLUSH_PROBE, %g5 | |
228 | add %o4, %o5, %g1 | |
229 | orcc %g5, 0, %g0 | |
230 | be 2f | |
231 | add %o4, %g1, %g2 | |
232 | add %o4, %g2, %g3 | |
233 | sub %o0, -PAGE_SIZE, %o0 | |
234 | add %o4, %g3, %g4 | |
235 | add %o4, %g4, %g5 | |
236 | add %o4, %g5, %g7 | |
237 | ||
238 | /* BLAMMO! */ | |
239 | 1: | |
240 | sub %o0, %g7, %o0 | |
241 | sta %g0, [%o0 + %g0] ASI_M_FLUSH_PAGE | |
242 | sta %g0, [%o0 + %o4] ASI_M_FLUSH_PAGE | |
243 | sta %g0, [%o0 + %o5] ASI_M_FLUSH_PAGE | |
244 | sta %g0, [%o0 + %g1] ASI_M_FLUSH_PAGE | |
245 | sta %g0, [%o0 + %g2] ASI_M_FLUSH_PAGE | |
246 | sta %g0, [%o0 + %g3] ASI_M_FLUSH_PAGE | |
247 | andcc %o0, 0xffc, %g0 | |
248 | sta %g0, [%o0 + %g4] ASI_M_FLUSH_PAGE | |
249 | bne 1b | |
250 | sta %g0, [%o0 + %g5] ASI_M_FLUSH_PAGE | |
251 | 2: | |
252 | mov SRMMU_FAULT_STATUS, %g1 | |
253 | retl | |
254 | lda [%g1] ASI_M_MMUREGS, %g0 | |
255 | ||
256 | /* HyperSparc is IO cache coherent. */ | |
257 | hypersparc_flush_page_for_dma: | |
258 | retl | |
259 | nop | |
260 | ||
261 | /* It was noted that at boot time a TLB flush all in a delay slot | |
262 | * can deliver an illegal instruction to the processor if the timing | |
263 | * is just right... | |
264 | */ | |
265 | hypersparc_flush_tlb_all: | |
266 | mov 0x400, %g1 | |
267 | sta %g0, [%g1] ASI_M_FLUSH_PROBE | |
268 | retl | |
269 | nop | |
270 | ||
271 | hypersparc_flush_tlb_mm: | |
272 | mov SRMMU_CTX_REG, %g1 | |
273 | ld [%o0 + AOFF_mm_context], %o1 | |
274 | lda [%g1] ASI_M_MMUREGS, %g5 | |
275 | #ifndef CONFIG_SMP | |
276 | cmp %o1, -1 | |
277 | be hypersparc_flush_tlb_mm_out | |
278 | #endif | |
279 | mov 0x300, %g2 | |
280 | sta %o1, [%g1] ASI_M_MMUREGS | |
281 | sta %g0, [%g2] ASI_M_FLUSH_PROBE | |
282 | hypersparc_flush_tlb_mm_out: | |
283 | retl | |
284 | sta %g5, [%g1] ASI_M_MMUREGS | |
285 | ||
286 | hypersparc_flush_tlb_range: | |
961246b4 | 287 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
288 | mov SRMMU_CTX_REG, %g1 |
289 | ld [%o0 + AOFF_mm_context], %o3 | |
290 | lda [%g1] ASI_M_MMUREGS, %g5 | |
291 | #ifndef CONFIG_SMP | |
292 | cmp %o3, -1 | |
293 | be hypersparc_flush_tlb_range_out | |
294 | #endif | |
295 | sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4 | |
296 | sta %o3, [%g1] ASI_M_MMUREGS | |
297 | and %o1, %o4, %o1 | |
298 | add %o1, 0x200, %o1 | |
299 | sta %g0, [%o1] ASI_M_FLUSH_PROBE | |
300 | 1: | |
301 | sub %o1, %o4, %o1 | |
302 | cmp %o1, %o2 | |
303 | blu,a 1b | |
304 | sta %g0, [%o1] ASI_M_FLUSH_PROBE | |
305 | hypersparc_flush_tlb_range_out: | |
306 | retl | |
307 | sta %g5, [%g1] ASI_M_MMUREGS | |
308 | ||
309 | hypersparc_flush_tlb_page: | |
961246b4 | 310 | ld [%o0 + VMA_VM_MM], %o0 |
1da177e4 LT |
311 | mov SRMMU_CTX_REG, %g1 |
312 | ld [%o0 + AOFF_mm_context], %o3 | |
313 | andn %o1, (PAGE_SIZE - 1), %o1 | |
314 | #ifndef CONFIG_SMP | |
315 | cmp %o3, -1 | |
316 | be hypersparc_flush_tlb_page_out | |
317 | #endif | |
318 | lda [%g1] ASI_M_MMUREGS, %g5 | |
319 | sta %o3, [%g1] ASI_M_MMUREGS | |
320 | sta %g0, [%o1] ASI_M_FLUSH_PROBE | |
321 | hypersparc_flush_tlb_page_out: | |
322 | retl | |
323 | sta %g5, [%g1] ASI_M_MMUREGS | |
324 | ||
325 | __INIT | |
326 | ||
327 | /* High speed page clear/copy. */ | |
328 | hypersparc_bzero_1page: | |
329 | /* NOTE: This routine has to be shorter than 40insns --jj */ | |
330 | clr %g1 | |
331 | mov 32, %g2 | |
332 | mov 64, %g3 | |
333 | mov 96, %g4 | |
334 | mov 128, %g5 | |
335 | mov 160, %g7 | |
336 | mov 192, %o2 | |
337 | mov 224, %o3 | |
338 | mov 16, %o1 | |
339 | 1: | |
340 | stda %g0, [%o0 + %g0] ASI_M_BFILL | |
341 | stda %g0, [%o0 + %g2] ASI_M_BFILL | |
342 | stda %g0, [%o0 + %g3] ASI_M_BFILL | |
343 | stda %g0, [%o0 + %g4] ASI_M_BFILL | |
344 | stda %g0, [%o0 + %g5] ASI_M_BFILL | |
345 | stda %g0, [%o0 + %g7] ASI_M_BFILL | |
346 | stda %g0, [%o0 + %o2] ASI_M_BFILL | |
347 | stda %g0, [%o0 + %o3] ASI_M_BFILL | |
348 | subcc %o1, 1, %o1 | |
349 | bne 1b | |
350 | add %o0, 256, %o0 | |
351 | ||
352 | retl | |
353 | nop | |
354 | ||
355 | hypersparc_copy_1page: | |
356 | /* NOTE: This routine has to be shorter than 70insns --jj */ | |
357 | sub %o1, %o0, %o2 ! difference | |
358 | mov 16, %g1 | |
359 | 1: | |
360 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
361 | add %o0, 32, %o0 | |
362 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
363 | add %o0, 32, %o0 | |
364 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
365 | add %o0, 32, %o0 | |
366 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
367 | add %o0, 32, %o0 | |
368 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
369 | add %o0, 32, %o0 | |
370 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
371 | add %o0, 32, %o0 | |
372 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
373 | add %o0, 32, %o0 | |
374 | sta %o0, [%o0 + %o2] ASI_M_BCOPY | |
375 | subcc %g1, 1, %g1 | |
376 | bne 1b | |
377 | add %o0, 32, %o0 | |
378 | ||
379 | retl | |
380 | nop | |
381 | ||
382 | .globl hypersparc_setup_blockops | |
383 | hypersparc_setup_blockops: | |
384 | sethi %hi(bzero_1page), %o0 | |
385 | or %o0, %lo(bzero_1page), %o0 | |
386 | sethi %hi(hypersparc_bzero_1page), %o1 | |
387 | or %o1, %lo(hypersparc_bzero_1page), %o1 | |
388 | sethi %hi(hypersparc_copy_1page), %o2 | |
389 | or %o2, %lo(hypersparc_copy_1page), %o2 | |
390 | ld [%o1], %o4 | |
391 | 1: | |
392 | add %o1, 4, %o1 | |
393 | st %o4, [%o0] | |
394 | add %o0, 4, %o0 | |
395 | cmp %o1, %o2 | |
396 | bne 1b | |
397 | ld [%o1], %o4 | |
398 | sethi %hi(__copy_1page), %o0 | |
399 | or %o0, %lo(__copy_1page), %o0 | |
400 | sethi %hi(hypersparc_setup_blockops), %o2 | |
401 | or %o2, %lo(hypersparc_setup_blockops), %o2 | |
402 | ld [%o1], %o4 | |
403 | 1: | |
404 | add %o1, 4, %o1 | |
405 | st %o4, [%o0] | |
406 | add %o0, 4, %o0 | |
407 | cmp %o1, %o2 | |
408 | bne 1b | |
409 | ld [%o1], %o4 | |
410 | sta %g0, [%g0] ASI_M_FLUSH_IWHOLE | |
411 | retl | |
412 | nop |