Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Inline assembly cache operations. | |
7 | * | |
79add627 | 8 | * Copyright (C) 1996 David S. Miller (davem@davemloft.net) |
1da177e4 LT |
9 | * Copyright (C) 1997 - 2002 Ralf Baechle (ralf@gnu.org) |
10 | * Copyright (C) 2004 Ralf Baechle (ralf@linux-mips.org) | |
11 | */ | |
12 | #ifndef _ASM_R4KCACHE_H | |
13 | #define _ASM_R4KCACHE_H | |
14 | ||
15 | #include <asm/asm.h> | |
16 | #include <asm/cacheops.h> | |
41700e73 | 17 | #include <asm/cpu-features.h> |
14bd8c08 | 18 | #include <asm/cpu-type.h> |
41c594ab | 19 | #include <asm/mipsmtregs.h> |
de8974e3 | 20 | #include <asm/uaccess.h> /* for segment_eq() */ |
1da177e4 | 21 | |
d116e812 DCZ |
22 | extern void (*r4k_blast_dcache)(void); |
23 | extern void (*r4k_blast_icache)(void); | |
24 | ||
1da177e4 LT |
25 | /* |
26 | * This macro return a properly sign-extended address suitable as base address | |
27 | * for indexed cache operations. Two issues here: | |
28 | * | |
29 | * - The MIPS32 and MIPS64 specs permit an implementation to directly derive | |
70342287 RB |
30 | * the index bits from the virtual address. This breaks with tradition |
31 | * set by the R4000. To keep unpleasant surprises from happening we pick | |
1da177e4 | 32 | * an address in KSEG0 / CKSEG0. |
70342287 | 33 | * - We need a properly sign extended address for 64-bit code. To get away |
1da177e4 LT |
34 | * without ifdefs we let the compiler do it by a type cast. |
35 | */ | |
36 | #define INDEX_BASE CKSEG0 | |
37 | ||
38 | #define cache_op(op,addr) \ | |
39 | __asm__ __volatile__( \ | |
2fe25f67 | 40 | " .set push \n" \ |
1da177e4 | 41 | " .set noreorder \n" \ |
a809d460 | 42 | " .set arch=r4000 \n" \ |
1da177e4 | 43 | " cache %0, %1 \n" \ |
2fe25f67 | 44 | " .set pop \n" \ |
1da177e4 | 45 | : \ |
675055bf | 46 | : "i" (op), "R" (*(unsigned char *)(addr))) |
1da177e4 | 47 | |
41c594ab | 48 | #ifdef CONFIG_MIPS_MT |
b633648c | 49 | |
41c594ab RB |
50 | #define __iflush_prologue \ |
51 | unsigned long redundance; \ | |
52 | extern int mt_n_iflushes; \ | |
41c594ab RB |
53 | for (redundance = 0; redundance < mt_n_iflushes; redundance++) { |
54 | ||
55 | #define __iflush_epilogue \ | |
41c594ab RB |
56 | } |
57 | ||
58 | #define __dflush_prologue \ | |
59 | unsigned long redundance; \ | |
60 | extern int mt_n_dflushes; \ | |
41c594ab RB |
61 | for (redundance = 0; redundance < mt_n_dflushes; redundance++) { |
62 | ||
63 | #define __dflush_epilogue \ | |
41c594ab RB |
64 | } |
65 | ||
66 | #define __inv_dflush_prologue __dflush_prologue | |
67 | #define __inv_dflush_epilogue __dflush_epilogue | |
68 | #define __sflush_prologue { | |
69 | #define __sflush_epilogue } | |
70 | #define __inv_sflush_prologue __sflush_prologue | |
71 | #define __inv_sflush_epilogue __sflush_epilogue | |
72 | ||
73 | #else /* CONFIG_MIPS_MT */ | |
74 | ||
75 | #define __iflush_prologue { | |
76 | #define __iflush_epilogue } | |
77 | #define __dflush_prologue { | |
78 | #define __dflush_epilogue } | |
79 | #define __inv_dflush_prologue { | |
80 | #define __inv_dflush_epilogue } | |
81 | #define __sflush_prologue { | |
82 | #define __sflush_epilogue } | |
83 | #define __inv_sflush_prologue { | |
84 | #define __inv_sflush_epilogue } | |
85 | ||
86 | #endif /* CONFIG_MIPS_MT */ | |
87 | ||
1da177e4 LT |
88 | static inline void flush_icache_line_indexed(unsigned long addr) |
89 | { | |
41c594ab | 90 | __iflush_prologue |
1da177e4 | 91 | cache_op(Index_Invalidate_I, addr); |
41c594ab | 92 | __iflush_epilogue |
1da177e4 LT |
93 | } |
94 | ||
95 | static inline void flush_dcache_line_indexed(unsigned long addr) | |
96 | { | |
41c594ab | 97 | __dflush_prologue |
1da177e4 | 98 | cache_op(Index_Writeback_Inv_D, addr); |
41c594ab | 99 | __dflush_epilogue |
1da177e4 LT |
100 | } |
101 | ||
102 | static inline void flush_scache_line_indexed(unsigned long addr) | |
103 | { | |
104 | cache_op(Index_Writeback_Inv_SD, addr); | |
105 | } | |
106 | ||
107 | static inline void flush_icache_line(unsigned long addr) | |
108 | { | |
41c594ab | 109 | __iflush_prologue |
14bd8c08 RB |
110 | switch (boot_cpu_type()) { |
111 | case CPU_LOONGSON2: | |
bad009fe | 112 | cache_op(Hit_Invalidate_I_Loongson2, addr); |
14bd8c08 RB |
113 | break; |
114 | ||
115 | default: | |
116 | cache_op(Hit_Invalidate_I, addr); | |
117 | break; | |
118 | } | |
41c594ab | 119 | __iflush_epilogue |
1da177e4 LT |
120 | } |
121 | ||
122 | static inline void flush_dcache_line(unsigned long addr) | |
123 | { | |
41c594ab | 124 | __dflush_prologue |
1da177e4 | 125 | cache_op(Hit_Writeback_Inv_D, addr); |
41c594ab | 126 | __dflush_epilogue |
1da177e4 LT |
127 | } |
128 | ||
129 | static inline void invalidate_dcache_line(unsigned long addr) | |
130 | { | |
41c594ab | 131 | __dflush_prologue |
1da177e4 | 132 | cache_op(Hit_Invalidate_D, addr); |
41c594ab | 133 | __dflush_epilogue |
1da177e4 LT |
134 | } |
135 | ||
136 | static inline void invalidate_scache_line(unsigned long addr) | |
137 | { | |
138 | cache_op(Hit_Invalidate_SD, addr); | |
139 | } | |
140 | ||
141 | static inline void flush_scache_line(unsigned long addr) | |
142 | { | |
143 | cache_op(Hit_Writeback_Inv_SD, addr); | |
144 | } | |
145 | ||
41700e73 AN |
146 | #define protected_cache_op(op,addr) \ |
147 | __asm__ __volatile__( \ | |
148 | " .set push \n" \ | |
149 | " .set noreorder \n" \ | |
a809d460 | 150 | " .set arch=r4000 \n" \ |
41700e73 AN |
151 | "1: cache %0, (%1) \n" \ |
152 | "2: .set pop \n" \ | |
153 | " .section __ex_table,\"a\" \n" \ | |
154 | " "STR(PTR)" 1b, 2b \n" \ | |
155 | " .previous" \ | |
156 | : \ | |
157 | : "i" (op), "r" (addr)) | |
158 | ||
a8053854 LY |
159 | #define protected_cachee_op(op,addr) \ |
160 | __asm__ __volatile__( \ | |
161 | " .set push \n" \ | |
162 | " .set noreorder \n" \ | |
163 | " .set mips0 \n" \ | |
164 | " .set eva \n" \ | |
165 | "1: cachee %0, (%1) \n" \ | |
166 | "2: .set pop \n" \ | |
167 | " .section __ex_table,\"a\" \n" \ | |
168 | " "STR(PTR)" 1b, 2b \n" \ | |
169 | " .previous" \ | |
170 | : \ | |
171 | : "i" (op), "r" (addr)) | |
172 | ||
1da177e4 LT |
173 | /* |
174 | * The next two are for badland addresses like signal trampolines. | |
175 | */ | |
176 | static inline void protected_flush_icache_line(unsigned long addr) | |
177 | { | |
14bd8c08 RB |
178 | switch (boot_cpu_type()) { |
179 | case CPU_LOONGSON2: | |
bad009fe | 180 | protected_cache_op(Hit_Invalidate_I_Loongson2, addr); |
14bd8c08 RB |
181 | break; |
182 | ||
183 | default: | |
a8053854 LY |
184 | #ifdef CONFIG_EVA |
185 | protected_cachee_op(Hit_Invalidate_I, addr); | |
186 | #else | |
14bd8c08 | 187 | protected_cache_op(Hit_Invalidate_I, addr); |
a8053854 | 188 | #endif |
14bd8c08 RB |
189 | break; |
190 | } | |
1da177e4 LT |
191 | } |
192 | ||
193 | /* | |
194 | * R10000 / R12000 hazard - these processors don't support the Hit_Writeback_D | |
195 | * cacheop so we use Hit_Writeback_Inv_D which is supported by all R4000-style | |
196 | * caches. We're talking about one cacheline unnecessarily getting invalidated | |
2fe25f67 | 197 | * here so the penalty isn't overly hard. |
1da177e4 LT |
198 | */ |
199 | static inline void protected_writeback_dcache_line(unsigned long addr) | |
200 | { | |
83fd4344 MC |
201 | #ifdef CONFIG_EVA |
202 | protected_cachee_op(Hit_Writeback_Inv_D, addr); | |
203 | #else | |
41700e73 | 204 | protected_cache_op(Hit_Writeback_Inv_D, addr); |
83fd4344 | 205 | #endif |
1da177e4 LT |
206 | } |
207 | ||
208 | static inline void protected_writeback_scache_line(unsigned long addr) | |
209 | { | |
41700e73 | 210 | protected_cache_op(Hit_Writeback_Inv_SD, addr); |
1da177e4 LT |
211 | } |
212 | ||
213 | /* | |
214 | * This one is RM7000-specific | |
215 | */ | |
216 | static inline void invalidate_tcache_page(unsigned long addr) | |
217 | { | |
218 | cache_op(Page_Invalidate_T, addr); | |
219 | } | |
220 | ||
221 | #define cache16_unroll32(base,op) \ | |
222 | __asm__ __volatile__( \ | |
2fe25f67 | 223 | " .set push \n" \ |
1da177e4 LT |
224 | " .set noreorder \n" \ |
225 | " .set mips3 \n" \ | |
226 | " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \ | |
227 | " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \ | |
228 | " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \ | |
229 | " cache %1, 0x060(%0); cache %1, 0x070(%0) \n" \ | |
230 | " cache %1, 0x080(%0); cache %1, 0x090(%0) \n" \ | |
231 | " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0) \n" \ | |
232 | " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0) \n" \ | |
233 | " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0) \n" \ | |
234 | " cache %1, 0x100(%0); cache %1, 0x110(%0) \n" \ | |
235 | " cache %1, 0x120(%0); cache %1, 0x130(%0) \n" \ | |
236 | " cache %1, 0x140(%0); cache %1, 0x150(%0) \n" \ | |
237 | " cache %1, 0x160(%0); cache %1, 0x170(%0) \n" \ | |
238 | " cache %1, 0x180(%0); cache %1, 0x190(%0) \n" \ | |
239 | " cache %1, 0x1a0(%0); cache %1, 0x1b0(%0) \n" \ | |
240 | " cache %1, 0x1c0(%0); cache %1, 0x1d0(%0) \n" \ | |
241 | " cache %1, 0x1e0(%0); cache %1, 0x1f0(%0) \n" \ | |
2fe25f67 | 242 | " .set pop \n" \ |
1da177e4 LT |
243 | : \ |
244 | : "r" (base), \ | |
245 | "i" (op)); | |
246 | ||
1da177e4 LT |
247 | #define cache32_unroll32(base,op) \ |
248 | __asm__ __volatile__( \ | |
2fe25f67 | 249 | " .set push \n" \ |
1da177e4 LT |
250 | " .set noreorder \n" \ |
251 | " .set mips3 \n" \ | |
252 | " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \ | |
253 | " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \ | |
254 | " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \ | |
255 | " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0) \n" \ | |
256 | " cache %1, 0x100(%0); cache %1, 0x120(%0) \n" \ | |
257 | " cache %1, 0x140(%0); cache %1, 0x160(%0) \n" \ | |
258 | " cache %1, 0x180(%0); cache %1, 0x1a0(%0) \n" \ | |
259 | " cache %1, 0x1c0(%0); cache %1, 0x1e0(%0) \n" \ | |
260 | " cache %1, 0x200(%0); cache %1, 0x220(%0) \n" \ | |
261 | " cache %1, 0x240(%0); cache %1, 0x260(%0) \n" \ | |
262 | " cache %1, 0x280(%0); cache %1, 0x2a0(%0) \n" \ | |
263 | " cache %1, 0x2c0(%0); cache %1, 0x2e0(%0) \n" \ | |
264 | " cache %1, 0x300(%0); cache %1, 0x320(%0) \n" \ | |
265 | " cache %1, 0x340(%0); cache %1, 0x360(%0) \n" \ | |
266 | " cache %1, 0x380(%0); cache %1, 0x3a0(%0) \n" \ | |
267 | " cache %1, 0x3c0(%0); cache %1, 0x3e0(%0) \n" \ | |
2fe25f67 | 268 | " .set pop \n" \ |
1da177e4 LT |
269 | : \ |
270 | : "r" (base), \ | |
271 | "i" (op)); | |
272 | ||
1da177e4 LT |
273 | #define cache64_unroll32(base,op) \ |
274 | __asm__ __volatile__( \ | |
2fe25f67 | 275 | " .set push \n" \ |
1da177e4 LT |
276 | " .set noreorder \n" \ |
277 | " .set mips3 \n" \ | |
278 | " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \ | |
279 | " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \ | |
280 | " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \ | |
281 | " cache %1, 0x180(%0); cache %1, 0x1c0(%0) \n" \ | |
282 | " cache %1, 0x200(%0); cache %1, 0x240(%0) \n" \ | |
283 | " cache %1, 0x280(%0); cache %1, 0x2c0(%0) \n" \ | |
284 | " cache %1, 0x300(%0); cache %1, 0x340(%0) \n" \ | |
285 | " cache %1, 0x380(%0); cache %1, 0x3c0(%0) \n" \ | |
286 | " cache %1, 0x400(%0); cache %1, 0x440(%0) \n" \ | |
287 | " cache %1, 0x480(%0); cache %1, 0x4c0(%0) \n" \ | |
288 | " cache %1, 0x500(%0); cache %1, 0x540(%0) \n" \ | |
289 | " cache %1, 0x580(%0); cache %1, 0x5c0(%0) \n" \ | |
290 | " cache %1, 0x600(%0); cache %1, 0x640(%0) \n" \ | |
291 | " cache %1, 0x680(%0); cache %1, 0x6c0(%0) \n" \ | |
292 | " cache %1, 0x700(%0); cache %1, 0x740(%0) \n" \ | |
293 | " cache %1, 0x780(%0); cache %1, 0x7c0(%0) \n" \ | |
2fe25f67 | 294 | " .set pop \n" \ |
1da177e4 LT |
295 | : \ |
296 | : "r" (base), \ | |
297 | "i" (op)); | |
298 | ||
1da177e4 LT |
299 | #define cache128_unroll32(base,op) \ |
300 | __asm__ __volatile__( \ | |
2fe25f67 | 301 | " .set push \n" \ |
1da177e4 LT |
302 | " .set noreorder \n" \ |
303 | " .set mips3 \n" \ | |
304 | " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \ | |
305 | " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \ | |
306 | " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \ | |
307 | " cache %1, 0x300(%0); cache %1, 0x380(%0) \n" \ | |
308 | " cache %1, 0x400(%0); cache %1, 0x480(%0) \n" \ | |
309 | " cache %1, 0x500(%0); cache %1, 0x580(%0) \n" \ | |
310 | " cache %1, 0x600(%0); cache %1, 0x680(%0) \n" \ | |
311 | " cache %1, 0x700(%0); cache %1, 0x780(%0) \n" \ | |
312 | " cache %1, 0x800(%0); cache %1, 0x880(%0) \n" \ | |
313 | " cache %1, 0x900(%0); cache %1, 0x980(%0) \n" \ | |
314 | " cache %1, 0xa00(%0); cache %1, 0xa80(%0) \n" \ | |
315 | " cache %1, 0xb00(%0); cache %1, 0xb80(%0) \n" \ | |
316 | " cache %1, 0xc00(%0); cache %1, 0xc80(%0) \n" \ | |
317 | " cache %1, 0xd00(%0); cache %1, 0xd80(%0) \n" \ | |
318 | " cache %1, 0xe00(%0); cache %1, 0xe80(%0) \n" \ | |
319 | " cache %1, 0xf00(%0); cache %1, 0xf80(%0) \n" \ | |
2fe25f67 | 320 | " .set pop \n" \ |
1da177e4 LT |
321 | : \ |
322 | : "r" (base), \ | |
323 | "i" (op)); | |
324 | ||
de8974e3 LY |
325 | /* |
326 | * Perform the cache operation specified by op using a user mode virtual | |
327 | * address while in kernel mode. | |
328 | */ | |
329 | #define cache16_unroll32_user(base,op) \ | |
330 | __asm__ __volatile__( \ | |
331 | " .set push \n" \ | |
332 | " .set noreorder \n" \ | |
333 | " .set mips0 \n" \ | |
334 | " .set eva \n" \ | |
335 | " cachee %1, 0x000(%0); cachee %1, 0x010(%0) \n" \ | |
336 | " cachee %1, 0x020(%0); cachee %1, 0x030(%0) \n" \ | |
337 | " cachee %1, 0x040(%0); cachee %1, 0x050(%0) \n" \ | |
338 | " cachee %1, 0x060(%0); cachee %1, 0x070(%0) \n" \ | |
339 | " cachee %1, 0x080(%0); cachee %1, 0x090(%0) \n" \ | |
340 | " cachee %1, 0x0a0(%0); cachee %1, 0x0b0(%0) \n" \ | |
341 | " cachee %1, 0x0c0(%0); cachee %1, 0x0d0(%0) \n" \ | |
342 | " cachee %1, 0x0e0(%0); cachee %1, 0x0f0(%0) \n" \ | |
343 | " cachee %1, 0x100(%0); cachee %1, 0x110(%0) \n" \ | |
344 | " cachee %1, 0x120(%0); cachee %1, 0x130(%0) \n" \ | |
345 | " cachee %1, 0x140(%0); cachee %1, 0x150(%0) \n" \ | |
346 | " cachee %1, 0x160(%0); cachee %1, 0x170(%0) \n" \ | |
347 | " cachee %1, 0x180(%0); cachee %1, 0x190(%0) \n" \ | |
348 | " cachee %1, 0x1a0(%0); cachee %1, 0x1b0(%0) \n" \ | |
349 | " cachee %1, 0x1c0(%0); cachee %1, 0x1d0(%0) \n" \ | |
350 | " cachee %1, 0x1e0(%0); cachee %1, 0x1f0(%0) \n" \ | |
351 | " .set pop \n" \ | |
352 | : \ | |
353 | : "r" (base), \ | |
354 | "i" (op)); | |
355 | ||
356 | #define cache32_unroll32_user(base, op) \ | |
357 | __asm__ __volatile__( \ | |
358 | " .set push \n" \ | |
359 | " .set noreorder \n" \ | |
360 | " .set mips0 \n" \ | |
361 | " .set eva \n" \ | |
362 | " cachee %1, 0x000(%0); cachee %1, 0x020(%0) \n" \ | |
363 | " cachee %1, 0x040(%0); cachee %1, 0x060(%0) \n" \ | |
364 | " cachee %1, 0x080(%0); cachee %1, 0x0a0(%0) \n" \ | |
365 | " cachee %1, 0x0c0(%0); cachee %1, 0x0e0(%0) \n" \ | |
366 | " cachee %1, 0x100(%0); cachee %1, 0x120(%0) \n" \ | |
367 | " cachee %1, 0x140(%0); cachee %1, 0x160(%0) \n" \ | |
368 | " cachee %1, 0x180(%0); cachee %1, 0x1a0(%0) \n" \ | |
369 | " cachee %1, 0x1c0(%0); cachee %1, 0x1e0(%0) \n" \ | |
370 | " cachee %1, 0x200(%0); cachee %1, 0x220(%0) \n" \ | |
371 | " cachee %1, 0x240(%0); cachee %1, 0x260(%0) \n" \ | |
372 | " cachee %1, 0x280(%0); cachee %1, 0x2a0(%0) \n" \ | |
373 | " cachee %1, 0x2c0(%0); cachee %1, 0x2e0(%0) \n" \ | |
374 | " cachee %1, 0x300(%0); cachee %1, 0x320(%0) \n" \ | |
375 | " cachee %1, 0x340(%0); cachee %1, 0x360(%0) \n" \ | |
376 | " cachee %1, 0x380(%0); cachee %1, 0x3a0(%0) \n" \ | |
377 | " cachee %1, 0x3c0(%0); cachee %1, 0x3e0(%0) \n" \ | |
378 | " .set pop \n" \ | |
379 | : \ | |
380 | : "r" (base), \ | |
381 | "i" (op)); | |
382 | ||
383 | #define cache64_unroll32_user(base, op) \ | |
384 | __asm__ __volatile__( \ | |
385 | " .set push \n" \ | |
386 | " .set noreorder \n" \ | |
387 | " .set mips0 \n" \ | |
388 | " .set eva \n" \ | |
389 | " cachee %1, 0x000(%0); cachee %1, 0x040(%0) \n" \ | |
390 | " cachee %1, 0x080(%0); cachee %1, 0x0c0(%0) \n" \ | |
391 | " cachee %1, 0x100(%0); cachee %1, 0x140(%0) \n" \ | |
392 | " cachee %1, 0x180(%0); cachee %1, 0x1c0(%0) \n" \ | |
393 | " cachee %1, 0x200(%0); cachee %1, 0x240(%0) \n" \ | |
394 | " cachee %1, 0x280(%0); cachee %1, 0x2c0(%0) \n" \ | |
395 | " cachee %1, 0x300(%0); cachee %1, 0x340(%0) \n" \ | |
396 | " cachee %1, 0x380(%0); cachee %1, 0x3c0(%0) \n" \ | |
397 | " cachee %1, 0x400(%0); cachee %1, 0x440(%0) \n" \ | |
398 | " cachee %1, 0x480(%0); cachee %1, 0x4c0(%0) \n" \ | |
399 | " cachee %1, 0x500(%0); cachee %1, 0x540(%0) \n" \ | |
400 | " cachee %1, 0x580(%0); cachee %1, 0x5c0(%0) \n" \ | |
401 | " cachee %1, 0x600(%0); cachee %1, 0x640(%0) \n" \ | |
402 | " cachee %1, 0x680(%0); cachee %1, 0x6c0(%0) \n" \ | |
403 | " cachee %1, 0x700(%0); cachee %1, 0x740(%0) \n" \ | |
404 | " cachee %1, 0x780(%0); cachee %1, 0x7c0(%0) \n" \ | |
405 | " .set pop \n" \ | |
406 | : \ | |
407 | : "r" (base), \ | |
408 | "i" (op)); | |
409 | ||
76f072a4 | 410 | /* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */ |
43a06847 AK |
411 | #define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize, extra) \ |
412 | static inline void extra##blast_##pfx##cache##lsize(void) \ | |
76f072a4 AN |
413 | { \ |
414 | unsigned long start = INDEX_BASE; \ | |
415 | unsigned long end = start + current_cpu_data.desc.waysize; \ | |
416 | unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \ | |
417 | unsigned long ws_end = current_cpu_data.desc.ways << \ | |
70342287 | 418 | current_cpu_data.desc.waybit; \ |
76f072a4 AN |
419 | unsigned long ws, addr; \ |
420 | \ | |
41c594ab RB |
421 | __##pfx##flush_prologue \ |
422 | \ | |
76f072a4 AN |
423 | for (ws = 0; ws < ws_end; ws += ws_inc) \ |
424 | for (addr = start; addr < end; addr += lsize * 32) \ | |
21a151d8 | 425 | cache##lsize##_unroll32(addr|ws, indexop); \ |
41c594ab RB |
426 | \ |
427 | __##pfx##flush_epilogue \ | |
76f072a4 AN |
428 | } \ |
429 | \ | |
43a06847 | 430 | static inline void extra##blast_##pfx##cache##lsize##_page(unsigned long page) \ |
76f072a4 AN |
431 | { \ |
432 | unsigned long start = page; \ | |
433 | unsigned long end = page + PAGE_SIZE; \ | |
434 | \ | |
41c594ab RB |
435 | __##pfx##flush_prologue \ |
436 | \ | |
76f072a4 | 437 | do { \ |
21a151d8 | 438 | cache##lsize##_unroll32(start, hitop); \ |
76f072a4 AN |
439 | start += lsize * 32; \ |
440 | } while (start < end); \ | |
41c594ab RB |
441 | \ |
442 | __##pfx##flush_epilogue \ | |
76f072a4 AN |
443 | } \ |
444 | \ | |
43a06847 | 445 | static inline void extra##blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \ |
76f072a4 | 446 | { \ |
de62893b AN |
447 | unsigned long indexmask = current_cpu_data.desc.waysize - 1; \ |
448 | unsigned long start = INDEX_BASE + (page & indexmask); \ | |
76f072a4 AN |
449 | unsigned long end = start + PAGE_SIZE; \ |
450 | unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \ | |
451 | unsigned long ws_end = current_cpu_data.desc.ways << \ | |
70342287 | 452 | current_cpu_data.desc.waybit; \ |
76f072a4 AN |
453 | unsigned long ws, addr; \ |
454 | \ | |
41c594ab RB |
455 | __##pfx##flush_prologue \ |
456 | \ | |
76f072a4 AN |
457 | for (ws = 0; ws < ws_end; ws += ws_inc) \ |
458 | for (addr = start; addr < end; addr += lsize * 32) \ | |
21a151d8 | 459 | cache##lsize##_unroll32(addr|ws, indexop); \ |
41c594ab RB |
460 | \ |
461 | __##pfx##flush_epilogue \ | |
76f072a4 AN |
462 | } |
463 | ||
43a06847 AK |
464 | __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16, ) |
465 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16, ) | |
466 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16, ) | |
467 | __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32, ) | |
468 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32, ) | |
469 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I_Loongson2, 32, loongson2_) | |
470 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32, ) | |
471 | __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 64, ) | |
472 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64, ) | |
473 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64, ) | |
18a8cd63 DD |
474 | __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 128, ) |
475 | __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 128, ) | |
43a06847 AK |
476 | __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128, ) |
477 | ||
478 | __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 16, ) | |
479 | __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 32, ) | |
480 | __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 16, ) | |
481 | __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 32, ) | |
482 | __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 64, ) | |
483 | __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 128, ) | |
e9c33572 | 484 | |
de8974e3 LY |
485 | #define __BUILD_BLAST_USER_CACHE(pfx, desc, indexop, hitop, lsize) \ |
486 | static inline void blast_##pfx##cache##lsize##_user_page(unsigned long page) \ | |
487 | { \ | |
488 | unsigned long start = page; \ | |
489 | unsigned long end = page + PAGE_SIZE; \ | |
490 | \ | |
491 | __##pfx##flush_prologue \ | |
492 | \ | |
493 | do { \ | |
494 | cache##lsize##_unroll32_user(start, hitop); \ | |
495 | start += lsize * 32; \ | |
496 | } while (start < end); \ | |
497 | \ | |
498 | __##pfx##flush_epilogue \ | |
499 | } | |
500 | ||
501 | __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, | |
502 | 16) | |
503 | __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16) | |
504 | __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, | |
505 | 32) | |
506 | __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32) | |
507 | __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, | |
508 | 64) | |
509 | __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64) | |
510 | ||
41700e73 | 511 | /* build blast_xxx_range, protected_blast_xxx_range */ |
14bd8c08 RB |
512 | #define __BUILD_BLAST_CACHE_RANGE(pfx, desc, hitop, prot, extra) \ |
513 | static inline void prot##extra##blast_##pfx##cache##_range(unsigned long start, \ | |
41700e73 AN |
514 | unsigned long end) \ |
515 | { \ | |
516 | unsigned long lsize = cpu_##desc##_line_size(); \ | |
517 | unsigned long addr = start & ~(lsize - 1); \ | |
518 | unsigned long aend = (end - 1) & ~(lsize - 1); \ | |
41c594ab RB |
519 | \ |
520 | __##pfx##flush_prologue \ | |
521 | \ | |
41700e73 AN |
522 | while (1) { \ |
523 | prot##cache_op(hitop, addr); \ | |
524 | if (addr == aend) \ | |
525 | break; \ | |
526 | addr += lsize; \ | |
527 | } \ | |
41c594ab RB |
528 | \ |
529 | __##pfx##flush_epilogue \ | |
41700e73 AN |
530 | } |
531 | ||
de8974e3 LY |
532 | #ifndef CONFIG_EVA |
533 | ||
14bd8c08 | 534 | __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_, ) |
14bd8c08 | 535 | __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_, ) |
de8974e3 LY |
536 | |
537 | #else | |
538 | ||
539 | #define __BUILD_PROT_BLAST_CACHE_RANGE(pfx, desc, hitop) \ | |
540 | static inline void protected_blast_##pfx##cache##_range(unsigned long start,\ | |
541 | unsigned long end) \ | |
542 | { \ | |
543 | unsigned long lsize = cpu_##desc##_line_size(); \ | |
544 | unsigned long addr = start & ~(lsize - 1); \ | |
545 | unsigned long aend = (end - 1) & ~(lsize - 1); \ | |
546 | \ | |
547 | __##pfx##flush_prologue \ | |
548 | \ | |
549 | if (segment_eq(get_fs(), USER_DS)) { \ | |
550 | while (1) { \ | |
551 | protected_cachee_op(hitop, addr); \ | |
552 | if (addr == aend) \ | |
553 | break; \ | |
554 | addr += lsize; \ | |
555 | } \ | |
556 | } else { \ | |
557 | while (1) { \ | |
558 | protected_cache_op(hitop, addr); \ | |
559 | if (addr == aend) \ | |
560 | break; \ | |
561 | addr += lsize; \ | |
562 | } \ | |
563 | \ | |
564 | } \ | |
565 | __##pfx##flush_epilogue \ | |
566 | } | |
567 | ||
568 | __BUILD_PROT_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D) | |
569 | __BUILD_PROT_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I) | |
570 | ||
571 | #endif | |
572 | __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_, ) | |
bad009fe HC |
573 | __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I_Loongson2, \ |
574 | protected_, loongson2_) | |
14bd8c08 | 575 | __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, , ) |
41e62b04 | 576 | __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, , ) |
14bd8c08 | 577 | __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, , ) |
41700e73 | 578 | /* blast_inv_dcache_range */ |
14bd8c08 RB |
579 | __BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, , ) |
580 | __BUILD_BLAST_CACHE_RANGE(inv_s, scache, Hit_Invalidate_SD, , ) | |
41700e73 | 581 | |
1da177e4 | 582 | #endif /* _ASM_R4KCACHE_H */ |