Commit | Line | Data |
---|---|---|
858dd5d4 | 1 | /* |
d116e812 DCZ |
2 | * This file is subject to the terms and conditions of the GNU General Public |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that | |
7 | * TLB handlers run from KSEG0 | |
8 | * | |
9 | * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. | |
10 | * Authors: Sanjay Lal <sanjayl@kymasys.com> | |
11 | */ | |
858dd5d4 | 12 | |
858dd5d4 SL |
13 | #include <linux/sched.h> |
14 | #include <linux/smp.h> | |
15 | #include <linux/mm.h> | |
16 | #include <linux/delay.h> | |
403015b3 | 17 | #include <linux/export.h> |
858dd5d4 | 18 | #include <linux/kvm_host.h> |
6d17c0d1 SL |
19 | #include <linux/srcu.h> |
20 | ||
858dd5d4 SL |
21 | #include <asm/cpu.h> |
22 | #include <asm/bootinfo.h> | |
23 | #include <asm/mmu_context.h> | |
24 | #include <asm/pgtable.h> | |
25 | #include <asm/cacheflush.h> | |
e36059e5 | 26 | #include <asm/tlb.h> |
858dd5d4 SL |
27 | |
28 | #undef CONFIG_MIPS_MT | |
29 | #include <asm/r4kcache.h> | |
30 | #define CONFIG_MIPS_MT | |
31 | ||
32 | #define KVM_GUEST_PC_TLB 0 | |
33 | #define KVM_GUEST_SP_TLB 1 | |
34 | ||
858dd5d4 | 35 | atomic_t kvm_mips_instance; |
cb1b447f | 36 | EXPORT_SYMBOL_GPL(kvm_mips_instance); |
858dd5d4 | 37 | |
403015b3 | 38 | static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu) |
858dd5d4 | 39 | { |
4edf00a4 PB |
40 | int cpu = smp_processor_id(); |
41 | ||
42 | return vcpu->arch.guest_kernel_asid[cpu] & | |
43 | cpu_asid_mask(&cpu_data[cpu]); | |
858dd5d4 SL |
44 | } |
45 | ||
403015b3 | 46 | static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu) |
858dd5d4 | 47 | { |
4edf00a4 PB |
48 | int cpu = smp_processor_id(); |
49 | ||
50 | return vcpu->arch.guest_user_asid[cpu] & | |
51 | cpu_asid_mask(&cpu_data[cpu]); | |
858dd5d4 SL |
52 | } |
53 | ||
bdb7ed86 | 54 | inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu) |
858dd5d4 SL |
55 | { |
56 | return vcpu->kvm->arch.commpage_tlb; | |
57 | } | |
58 | ||
d116e812 | 59 | /* Structure defining an tlb entry data set. */ |
858dd5d4 SL |
60 | |
61 | void kvm_mips_dump_host_tlbs(void) | |
62 | { | |
63 | unsigned long old_entryhi; | |
64 | unsigned long old_pagemask; | |
65 | struct kvm_mips_tlb tlb; | |
66 | unsigned long flags; | |
67 | int i; | |
68 | ||
69 | local_irq_save(flags); | |
70 | ||
71 | old_entryhi = read_c0_entryhi(); | |
72 | old_pagemask = read_c0_pagemask(); | |
73 | ||
6ad78a5c | 74 | kvm_info("HOST TLBs:\n"); |
4edf00a4 PB |
75 | kvm_info("ASID: %#lx\n", read_c0_entryhi() & |
76 | cpu_asid_mask(¤t_cpu_data)); | |
858dd5d4 SL |
77 | |
78 | for (i = 0; i < current_cpu_data.tlbsize; i++) { | |
79 | write_c0_index(i); | |
80 | mtc0_tlbw_hazard(); | |
81 | ||
82 | tlb_read(); | |
83 | tlbw_use_hazard(); | |
84 | ||
85 | tlb.tlb_hi = read_c0_entryhi(); | |
86 | tlb.tlb_lo0 = read_c0_entrylo0(); | |
87 | tlb.tlb_lo1 = read_c0_entrylo1(); | |
88 | tlb.tlb_mask = read_c0_pagemask(); | |
89 | ||
6ad78a5c DCZ |
90 | kvm_info("TLB%c%3d Hi 0x%08lx ", |
91 | (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*', | |
92 | i, tlb.tlb_hi); | |
8cffd197 JH |
93 | kvm_info("Lo0=0x%09llx %c%c attr %lx ", |
94 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo0), | |
6ad78a5c DCZ |
95 | (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ', |
96 | (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ', | |
97 | (tlb.tlb_lo0 >> 3) & 7); | |
8cffd197 JH |
98 | kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n", |
99 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo1), | |
6ad78a5c DCZ |
100 | (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ', |
101 | (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ', | |
102 | (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask); | |
858dd5d4 SL |
103 | } |
104 | write_c0_entryhi(old_entryhi); | |
105 | write_c0_pagemask(old_pagemask); | |
106 | mtc0_tlbw_hazard(); | |
107 | local_irq_restore(flags); | |
108 | } | |
cb1b447f | 109 | EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs); |
858dd5d4 SL |
110 | |
111 | void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu) | |
112 | { | |
113 | struct mips_coproc *cop0 = vcpu->arch.cop0; | |
114 | struct kvm_mips_tlb tlb; | |
115 | int i; | |
116 | ||
6ad78a5c DCZ |
117 | kvm_info("Guest TLBs:\n"); |
118 | kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0)); | |
858dd5d4 SL |
119 | |
120 | for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { | |
121 | tlb = vcpu->arch.guest_tlb[i]; | |
6ad78a5c DCZ |
122 | kvm_info("TLB%c%3d Hi 0x%08lx ", |
123 | (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*', | |
124 | i, tlb.tlb_hi); | |
8cffd197 JH |
125 | kvm_info("Lo0=0x%09llx %c%c attr %lx ", |
126 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo0), | |
6ad78a5c DCZ |
127 | (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ', |
128 | (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ', | |
129 | (tlb.tlb_lo0 >> 3) & 7); | |
8cffd197 JH |
130 | kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n", |
131 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo1), | |
6ad78a5c DCZ |
132 | (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ', |
133 | (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ', | |
134 | (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask); | |
858dd5d4 SL |
135 | } |
136 | } | |
cb1b447f | 137 | EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs); |
858dd5d4 | 138 | |
858dd5d4 SL |
139 | /* XXXKYMA: Must be called with interrupts disabled */ |
140 | /* set flush_dcache_mask == 0 if no dcache flush required */ | |
d116e812 DCZ |
141 | int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi, |
142 | unsigned long entrylo0, unsigned long entrylo1, | |
143 | int flush_dcache_mask) | |
858dd5d4 SL |
144 | { |
145 | unsigned long flags; | |
146 | unsigned long old_entryhi; | |
b045c406 | 147 | int idx; |
858dd5d4 SL |
148 | |
149 | local_irq_save(flags); | |
150 | ||
858dd5d4 SL |
151 | old_entryhi = read_c0_entryhi(); |
152 | write_c0_entryhi(entryhi); | |
153 | mtc0_tlbw_hazard(); | |
154 | ||
155 | tlb_probe(); | |
156 | tlb_probe_hazard(); | |
157 | idx = read_c0_index(); | |
158 | ||
159 | if (idx > current_cpu_data.tlbsize) { | |
160 | kvm_err("%s: Invalid Index: %d\n", __func__, idx); | |
161 | kvm_mips_dump_host_tlbs(); | |
cfec0e75 | 162 | local_irq_restore(flags); |
858dd5d4 SL |
163 | return -1; |
164 | } | |
165 | ||
858dd5d4 SL |
166 | write_c0_entrylo0(entrylo0); |
167 | write_c0_entrylo1(entrylo1); | |
168 | mtc0_tlbw_hazard(); | |
169 | ||
b5dfc6c1 JH |
170 | if (idx < 0) |
171 | tlb_write_random(); | |
172 | else | |
173 | tlb_write_indexed(); | |
858dd5d4 SL |
174 | tlbw_use_hazard(); |
175 | ||
3d654833 JH |
176 | kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n", |
177 | vcpu->arch.pc, idx, read_c0_entryhi(), | |
178 | read_c0_entrylo0(), read_c0_entrylo1()); | |
858dd5d4 SL |
179 | |
180 | /* Flush D-cache */ | |
181 | if (flush_dcache_mask) { | |
182 | if (entrylo0 & MIPS3_PG_V) { | |
183 | ++vcpu->stat.flush_dcache_exits; | |
d116e812 DCZ |
184 | flush_data_cache_page((entryhi & VPN2_MASK) & |
185 | ~flush_dcache_mask); | |
858dd5d4 SL |
186 | } |
187 | if (entrylo1 & MIPS3_PG_V) { | |
188 | ++vcpu->stat.flush_dcache_exits; | |
d116e812 DCZ |
189 | flush_data_cache_page(((entryhi & VPN2_MASK) & |
190 | ~flush_dcache_mask) | | |
191 | (0x1 << PAGE_SHIFT)); | |
858dd5d4 SL |
192 | } |
193 | } | |
194 | ||
195 | /* Restore old ASID */ | |
196 | write_c0_entryhi(old_entryhi); | |
197 | mtc0_tlbw_hazard(); | |
198 | tlbw_use_hazard(); | |
199 | local_irq_restore(flags); | |
200 | return 0; | |
201 | } | |
403015b3 | 202 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write); |
858dd5d4 SL |
203 | |
204 | int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr, | |
205 | struct kvm_vcpu *vcpu) | |
206 | { | |
ba049e93 | 207 | kvm_pfn_t pfn0, pfn1; |
858dd5d4 SL |
208 | unsigned long flags, old_entryhi = 0, vaddr = 0; |
209 | unsigned long entrylo0 = 0, entrylo1 = 0; | |
210 | ||
858dd5d4 SL |
211 | pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT; |
212 | pfn1 = 0; | |
d116e812 DCZ |
213 | entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | |
214 | (1 << 2) | (0x1 << 1); | |
858dd5d4 SL |
215 | entrylo1 = 0; |
216 | ||
217 | local_irq_save(flags); | |
218 | ||
219 | old_entryhi = read_c0_entryhi(); | |
220 | vaddr = badvaddr & (PAGE_MASK << 1); | |
221 | write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu)); | |
222 | mtc0_tlbw_hazard(); | |
223 | write_c0_entrylo0(entrylo0); | |
224 | mtc0_tlbw_hazard(); | |
225 | write_c0_entrylo1(entrylo1); | |
226 | mtc0_tlbw_hazard(); | |
227 | write_c0_index(kvm_mips_get_commpage_asid(vcpu)); | |
228 | mtc0_tlbw_hazard(); | |
229 | tlb_write_indexed(); | |
230 | mtc0_tlbw_hazard(); | |
231 | tlbw_use_hazard(); | |
232 | ||
d116e812 DCZ |
233 | kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n", |
234 | vcpu->arch.pc, read_c0_index(), read_c0_entryhi(), | |
235 | read_c0_entrylo0(), read_c0_entrylo1()); | |
858dd5d4 SL |
236 | |
237 | /* Restore old ASID */ | |
238 | write_c0_entryhi(old_entryhi); | |
239 | mtc0_tlbw_hazard(); | |
240 | tlbw_use_hazard(); | |
241 | local_irq_restore(flags); | |
242 | ||
243 | return 0; | |
244 | } | |
cb1b447f | 245 | EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault); |
858dd5d4 | 246 | |
858dd5d4 SL |
247 | int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi) |
248 | { | |
249 | int i; | |
250 | int index = -1; | |
251 | struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb; | |
252 | ||
858dd5d4 | 253 | for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { |
d116e812 DCZ |
254 | if (TLB_HI_VPN2_HIT(tlb[i], entryhi) && |
255 | TLB_HI_ASID_HIT(tlb[i], entryhi)) { | |
858dd5d4 SL |
256 | index = i; |
257 | break; | |
258 | } | |
259 | } | |
260 | ||
858dd5d4 SL |
261 | kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n", |
262 | __func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1); | |
858dd5d4 SL |
263 | |
264 | return index; | |
265 | } | |
cb1b447f | 266 | EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup); |
858dd5d4 SL |
267 | |
268 | int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr) | |
269 | { | |
270 | unsigned long old_entryhi, flags; | |
b045c406 | 271 | int idx; |
858dd5d4 | 272 | |
858dd5d4 SL |
273 | local_irq_save(flags); |
274 | ||
275 | old_entryhi = read_c0_entryhi(); | |
276 | ||
277 | if (KVM_GUEST_KERNEL_MODE(vcpu)) | |
d116e812 DCZ |
278 | write_c0_entryhi((vaddr & VPN2_MASK) | |
279 | kvm_mips_get_kernel_asid(vcpu)); | |
858dd5d4 | 280 | else { |
d116e812 DCZ |
281 | write_c0_entryhi((vaddr & VPN2_MASK) | |
282 | kvm_mips_get_user_asid(vcpu)); | |
858dd5d4 SL |
283 | } |
284 | ||
285 | mtc0_tlbw_hazard(); | |
286 | ||
287 | tlb_probe(); | |
288 | tlb_probe_hazard(); | |
289 | idx = read_c0_index(); | |
290 | ||
291 | /* Restore old ASID */ | |
292 | write_c0_entryhi(old_entryhi); | |
293 | mtc0_tlbw_hazard(); | |
294 | tlbw_use_hazard(); | |
295 | ||
296 | local_irq_restore(flags); | |
297 | ||
858dd5d4 | 298 | kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx); |
858dd5d4 SL |
299 | |
300 | return idx; | |
301 | } | |
cb1b447f | 302 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup); |
858dd5d4 SL |
303 | |
304 | int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va) | |
305 | { | |
306 | int idx; | |
307 | unsigned long flags, old_entryhi; | |
308 | ||
309 | local_irq_save(flags); | |
310 | ||
858dd5d4 SL |
311 | old_entryhi = read_c0_entryhi(); |
312 | ||
313 | write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu)); | |
314 | mtc0_tlbw_hazard(); | |
315 | ||
316 | tlb_probe(); | |
317 | tlb_probe_hazard(); | |
318 | idx = read_c0_index(); | |
319 | ||
320 | if (idx >= current_cpu_data.tlbsize) | |
321 | BUG(); | |
322 | ||
323 | if (idx > 0) { | |
324 | write_c0_entryhi(UNIQUE_ENTRYHI(idx)); | |
325 | mtc0_tlbw_hazard(); | |
326 | ||
327 | write_c0_entrylo0(0); | |
328 | mtc0_tlbw_hazard(); | |
329 | ||
330 | write_c0_entrylo1(0); | |
331 | mtc0_tlbw_hazard(); | |
332 | ||
333 | tlb_write_indexed(); | |
334 | mtc0_tlbw_hazard(); | |
335 | } | |
336 | ||
337 | write_c0_entryhi(old_entryhi); | |
338 | mtc0_tlbw_hazard(); | |
339 | tlbw_use_hazard(); | |
340 | ||
341 | local_irq_restore(flags); | |
342 | ||
3d654833 | 343 | if (idx > 0) |
858dd5d4 | 344 | kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__, |
3d654833 | 345 | (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx); |
858dd5d4 SL |
346 | |
347 | return 0; | |
348 | } | |
cb1b447f | 349 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv); |
858dd5d4 SL |
350 | |
351 | void kvm_mips_flush_host_tlb(int skip_kseg0) | |
352 | { | |
353 | unsigned long flags; | |
354 | unsigned long old_entryhi, entryhi; | |
355 | unsigned long old_pagemask; | |
356 | int entry = 0; | |
357 | int maxentry = current_cpu_data.tlbsize; | |
358 | ||
858dd5d4 SL |
359 | local_irq_save(flags); |
360 | ||
361 | old_entryhi = read_c0_entryhi(); | |
362 | old_pagemask = read_c0_pagemask(); | |
363 | ||
364 | /* Blast 'em all away. */ | |
365 | for (entry = 0; entry < maxentry; entry++) { | |
858dd5d4 SL |
366 | write_c0_index(entry); |
367 | mtc0_tlbw_hazard(); | |
368 | ||
369 | if (skip_kseg0) { | |
370 | tlb_read(); | |
371 | tlbw_use_hazard(); | |
372 | ||
373 | entryhi = read_c0_entryhi(); | |
374 | ||
375 | /* Don't blow away guest kernel entries */ | |
d116e812 | 376 | if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0) |
858dd5d4 | 377 | continue; |
858dd5d4 SL |
378 | } |
379 | ||
380 | /* Make sure all entries differ. */ | |
381 | write_c0_entryhi(UNIQUE_ENTRYHI(entry)); | |
382 | mtc0_tlbw_hazard(); | |
383 | write_c0_entrylo0(0); | |
384 | mtc0_tlbw_hazard(); | |
385 | write_c0_entrylo1(0); | |
386 | mtc0_tlbw_hazard(); | |
387 | ||
388 | tlb_write_indexed(); | |
389 | mtc0_tlbw_hazard(); | |
390 | } | |
391 | ||
392 | tlbw_use_hazard(); | |
393 | ||
394 | write_c0_entryhi(old_entryhi); | |
395 | write_c0_pagemask(old_pagemask); | |
396 | mtc0_tlbw_hazard(); | |
397 | tlbw_use_hazard(); | |
398 | ||
399 | local_irq_restore(flags); | |
400 | } | |
cb1b447f | 401 | EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb); |
858dd5d4 | 402 | |
858dd5d4 SL |
403 | void kvm_local_flush_tlb_all(void) |
404 | { | |
405 | unsigned long flags; | |
406 | unsigned long old_ctx; | |
407 | int entry = 0; | |
408 | ||
409 | local_irq_save(flags); | |
410 | /* Save old context and create impossible VPN2 value */ | |
411 | old_ctx = read_c0_entryhi(); | |
412 | write_c0_entrylo0(0); | |
413 | write_c0_entrylo1(0); | |
414 | ||
415 | /* Blast 'em all away. */ | |
416 | while (entry < current_cpu_data.tlbsize) { | |
417 | /* Make sure all entries differ. */ | |
418 | write_c0_entryhi(UNIQUE_ENTRYHI(entry)); | |
419 | write_c0_index(entry); | |
420 | mtc0_tlbw_hazard(); | |
421 | tlb_write_indexed(); | |
422 | entry++; | |
423 | } | |
424 | tlbw_use_hazard(); | |
425 | write_c0_entryhi(old_ctx); | |
426 | mtc0_tlbw_hazard(); | |
427 | ||
428 | local_irq_restore(flags); | |
429 | } | |
cb1b447f | 430 | EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all); |