Commit | Line | Data |
---|---|---|
858dd5d4 | 1 | /* |
d116e812 DCZ |
2 | * This file is subject to the terms and conditions of the GNU General Public |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that | |
7 | * TLB handlers run from KSEG0 | |
8 | * | |
9 | * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. | |
10 | * Authors: Sanjay Lal <sanjayl@kymasys.com> | |
11 | */ | |
858dd5d4 | 12 | |
858dd5d4 SL |
13 | #include <linux/sched.h> |
14 | #include <linux/smp.h> | |
15 | #include <linux/mm.h> | |
16 | #include <linux/delay.h> | |
403015b3 | 17 | #include <linux/export.h> |
858dd5d4 | 18 | #include <linux/kvm_host.h> |
6d17c0d1 SL |
19 | #include <linux/srcu.h> |
20 | ||
858dd5d4 SL |
21 | #include <asm/cpu.h> |
22 | #include <asm/bootinfo.h> | |
23 | #include <asm/mmu_context.h> | |
24 | #include <asm/pgtable.h> | |
25 | #include <asm/cacheflush.h> | |
e36059e5 | 26 | #include <asm/tlb.h> |
e922a4cb | 27 | #include <asm/tlbdebug.h> |
858dd5d4 SL |
28 | |
29 | #undef CONFIG_MIPS_MT | |
30 | #include <asm/r4kcache.h> | |
31 | #define CONFIG_MIPS_MT | |
32 | ||
33 | #define KVM_GUEST_PC_TLB 0 | |
34 | #define KVM_GUEST_SP_TLB 1 | |
35 | ||
858dd5d4 | 36 | atomic_t kvm_mips_instance; |
cb1b447f | 37 | EXPORT_SYMBOL_GPL(kvm_mips_instance); |
858dd5d4 | 38 | |
403015b3 | 39 | static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu) |
858dd5d4 | 40 | { |
4edf00a4 PB |
41 | int cpu = smp_processor_id(); |
42 | ||
43 | return vcpu->arch.guest_kernel_asid[cpu] & | |
44 | cpu_asid_mask(&cpu_data[cpu]); | |
858dd5d4 SL |
45 | } |
46 | ||
403015b3 | 47 | static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu) |
858dd5d4 | 48 | { |
4edf00a4 PB |
49 | int cpu = smp_processor_id(); |
50 | ||
51 | return vcpu->arch.guest_user_asid[cpu] & | |
52 | cpu_asid_mask(&cpu_data[cpu]); | |
858dd5d4 SL |
53 | } |
54 | ||
bdb7ed86 | 55 | inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu) |
858dd5d4 SL |
56 | { |
57 | return vcpu->kvm->arch.commpage_tlb; | |
58 | } | |
59 | ||
d116e812 | 60 | /* Structure defining an tlb entry data set. */ |
858dd5d4 SL |
61 | |
62 | void kvm_mips_dump_host_tlbs(void) | |
63 | { | |
858dd5d4 | 64 | unsigned long flags; |
858dd5d4 SL |
65 | |
66 | local_irq_save(flags); | |
67 | ||
6ad78a5c | 68 | kvm_info("HOST TLBs:\n"); |
e922a4cb JH |
69 | dump_tlb_regs(); |
70 | pr_info("\n"); | |
71 | dump_tlb_all(); | |
72 | ||
858dd5d4 SL |
73 | local_irq_restore(flags); |
74 | } | |
cb1b447f | 75 | EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs); |
858dd5d4 SL |
76 | |
77 | void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu) | |
78 | { | |
79 | struct mips_coproc *cop0 = vcpu->arch.cop0; | |
80 | struct kvm_mips_tlb tlb; | |
81 | int i; | |
82 | ||
6ad78a5c DCZ |
83 | kvm_info("Guest TLBs:\n"); |
84 | kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0)); | |
858dd5d4 SL |
85 | |
86 | for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { | |
87 | tlb = vcpu->arch.guest_tlb[i]; | |
6ad78a5c DCZ |
88 | kvm_info("TLB%c%3d Hi 0x%08lx ", |
89 | (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*', | |
90 | i, tlb.tlb_hi); | |
8cffd197 JH |
91 | kvm_info("Lo0=0x%09llx %c%c attr %lx ", |
92 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo0), | |
6ad78a5c DCZ |
93 | (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ', |
94 | (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ', | |
95 | (tlb.tlb_lo0 >> 3) & 7); | |
8cffd197 JH |
96 | kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n", |
97 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo1), | |
6ad78a5c DCZ |
98 | (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ', |
99 | (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ', | |
100 | (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask); | |
858dd5d4 SL |
101 | } |
102 | } | |
cb1b447f | 103 | EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs); |
858dd5d4 | 104 | |
858dd5d4 SL |
105 | /* XXXKYMA: Must be called with interrupts disabled */ |
106 | /* set flush_dcache_mask == 0 if no dcache flush required */ | |
d116e812 DCZ |
107 | int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi, |
108 | unsigned long entrylo0, unsigned long entrylo1, | |
109 | int flush_dcache_mask) | |
858dd5d4 SL |
110 | { |
111 | unsigned long flags; | |
112 | unsigned long old_entryhi; | |
b045c406 | 113 | int idx; |
858dd5d4 SL |
114 | |
115 | local_irq_save(flags); | |
116 | ||
858dd5d4 SL |
117 | old_entryhi = read_c0_entryhi(); |
118 | write_c0_entryhi(entryhi); | |
119 | mtc0_tlbw_hazard(); | |
120 | ||
121 | tlb_probe(); | |
122 | tlb_probe_hazard(); | |
123 | idx = read_c0_index(); | |
124 | ||
125 | if (idx > current_cpu_data.tlbsize) { | |
126 | kvm_err("%s: Invalid Index: %d\n", __func__, idx); | |
127 | kvm_mips_dump_host_tlbs(); | |
cfec0e75 | 128 | local_irq_restore(flags); |
858dd5d4 SL |
129 | return -1; |
130 | } | |
131 | ||
858dd5d4 SL |
132 | write_c0_entrylo0(entrylo0); |
133 | write_c0_entrylo1(entrylo1); | |
134 | mtc0_tlbw_hazard(); | |
135 | ||
b5dfc6c1 JH |
136 | if (idx < 0) |
137 | tlb_write_random(); | |
138 | else | |
139 | tlb_write_indexed(); | |
858dd5d4 SL |
140 | tlbw_use_hazard(); |
141 | ||
3d654833 JH |
142 | kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n", |
143 | vcpu->arch.pc, idx, read_c0_entryhi(), | |
144 | read_c0_entrylo0(), read_c0_entrylo1()); | |
858dd5d4 SL |
145 | |
146 | /* Flush D-cache */ | |
147 | if (flush_dcache_mask) { | |
148 | if (entrylo0 & MIPS3_PG_V) { | |
149 | ++vcpu->stat.flush_dcache_exits; | |
d116e812 DCZ |
150 | flush_data_cache_page((entryhi & VPN2_MASK) & |
151 | ~flush_dcache_mask); | |
858dd5d4 SL |
152 | } |
153 | if (entrylo1 & MIPS3_PG_V) { | |
154 | ++vcpu->stat.flush_dcache_exits; | |
d116e812 DCZ |
155 | flush_data_cache_page(((entryhi & VPN2_MASK) & |
156 | ~flush_dcache_mask) | | |
157 | (0x1 << PAGE_SHIFT)); | |
858dd5d4 SL |
158 | } |
159 | } | |
160 | ||
161 | /* Restore old ASID */ | |
162 | write_c0_entryhi(old_entryhi); | |
163 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
164 | local_irq_restore(flags); |
165 | return 0; | |
166 | } | |
403015b3 | 167 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write); |
858dd5d4 SL |
168 | |
169 | int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr, | |
170 | struct kvm_vcpu *vcpu) | |
171 | { | |
ba049e93 | 172 | kvm_pfn_t pfn0, pfn1; |
858dd5d4 SL |
173 | unsigned long flags, old_entryhi = 0, vaddr = 0; |
174 | unsigned long entrylo0 = 0, entrylo1 = 0; | |
175 | ||
858dd5d4 SL |
176 | pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT; |
177 | pfn1 = 0; | |
d116e812 DCZ |
178 | entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | |
179 | (1 << 2) | (0x1 << 1); | |
858dd5d4 SL |
180 | entrylo1 = 0; |
181 | ||
182 | local_irq_save(flags); | |
183 | ||
184 | old_entryhi = read_c0_entryhi(); | |
185 | vaddr = badvaddr & (PAGE_MASK << 1); | |
186 | write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu)); | |
858dd5d4 | 187 | write_c0_entrylo0(entrylo0); |
858dd5d4 | 188 | write_c0_entrylo1(entrylo1); |
858dd5d4 SL |
189 | write_c0_index(kvm_mips_get_commpage_asid(vcpu)); |
190 | mtc0_tlbw_hazard(); | |
191 | tlb_write_indexed(); | |
858dd5d4 SL |
192 | tlbw_use_hazard(); |
193 | ||
d116e812 DCZ |
194 | kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n", |
195 | vcpu->arch.pc, read_c0_index(), read_c0_entryhi(), | |
196 | read_c0_entrylo0(), read_c0_entrylo1()); | |
858dd5d4 SL |
197 | |
198 | /* Restore old ASID */ | |
199 | write_c0_entryhi(old_entryhi); | |
200 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
201 | local_irq_restore(flags); |
202 | ||
203 | return 0; | |
204 | } | |
cb1b447f | 205 | EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault); |
858dd5d4 | 206 | |
858dd5d4 SL |
207 | int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi) |
208 | { | |
209 | int i; | |
210 | int index = -1; | |
211 | struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb; | |
212 | ||
858dd5d4 | 213 | for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { |
d116e812 DCZ |
214 | if (TLB_HI_VPN2_HIT(tlb[i], entryhi) && |
215 | TLB_HI_ASID_HIT(tlb[i], entryhi)) { | |
858dd5d4 SL |
216 | index = i; |
217 | break; | |
218 | } | |
219 | } | |
220 | ||
858dd5d4 SL |
221 | kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n", |
222 | __func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1); | |
858dd5d4 SL |
223 | |
224 | return index; | |
225 | } | |
cb1b447f | 226 | EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup); |
858dd5d4 SL |
227 | |
228 | int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr) | |
229 | { | |
230 | unsigned long old_entryhi, flags; | |
b045c406 | 231 | int idx; |
858dd5d4 | 232 | |
858dd5d4 SL |
233 | local_irq_save(flags); |
234 | ||
235 | old_entryhi = read_c0_entryhi(); | |
236 | ||
237 | if (KVM_GUEST_KERNEL_MODE(vcpu)) | |
d116e812 DCZ |
238 | write_c0_entryhi((vaddr & VPN2_MASK) | |
239 | kvm_mips_get_kernel_asid(vcpu)); | |
858dd5d4 | 240 | else { |
d116e812 DCZ |
241 | write_c0_entryhi((vaddr & VPN2_MASK) | |
242 | kvm_mips_get_user_asid(vcpu)); | |
858dd5d4 SL |
243 | } |
244 | ||
245 | mtc0_tlbw_hazard(); | |
246 | ||
247 | tlb_probe(); | |
248 | tlb_probe_hazard(); | |
249 | idx = read_c0_index(); | |
250 | ||
251 | /* Restore old ASID */ | |
252 | write_c0_entryhi(old_entryhi); | |
253 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
254 | |
255 | local_irq_restore(flags); | |
256 | ||
858dd5d4 | 257 | kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx); |
858dd5d4 SL |
258 | |
259 | return idx; | |
260 | } | |
cb1b447f | 261 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup); |
858dd5d4 SL |
262 | |
263 | int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va) | |
264 | { | |
265 | int idx; | |
266 | unsigned long flags, old_entryhi; | |
267 | ||
268 | local_irq_save(flags); | |
269 | ||
858dd5d4 SL |
270 | old_entryhi = read_c0_entryhi(); |
271 | ||
272 | write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu)); | |
273 | mtc0_tlbw_hazard(); | |
274 | ||
275 | tlb_probe(); | |
276 | tlb_probe_hazard(); | |
277 | idx = read_c0_index(); | |
278 | ||
279 | if (idx >= current_cpu_data.tlbsize) | |
280 | BUG(); | |
281 | ||
282 | if (idx > 0) { | |
283 | write_c0_entryhi(UNIQUE_ENTRYHI(idx)); | |
858dd5d4 | 284 | write_c0_entrylo0(0); |
858dd5d4 SL |
285 | write_c0_entrylo1(0); |
286 | mtc0_tlbw_hazard(); | |
287 | ||
288 | tlb_write_indexed(); | |
138f7ad9 | 289 | tlbw_use_hazard(); |
858dd5d4 SL |
290 | } |
291 | ||
292 | write_c0_entryhi(old_entryhi); | |
293 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
294 | |
295 | local_irq_restore(flags); | |
296 | ||
3d654833 | 297 | if (idx > 0) |
858dd5d4 | 298 | kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__, |
3d654833 | 299 | (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx); |
858dd5d4 SL |
300 | |
301 | return 0; | |
302 | } | |
cb1b447f | 303 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv); |
858dd5d4 SL |
304 | |
305 | void kvm_mips_flush_host_tlb(int skip_kseg0) | |
306 | { | |
307 | unsigned long flags; | |
308 | unsigned long old_entryhi, entryhi; | |
309 | unsigned long old_pagemask; | |
310 | int entry = 0; | |
311 | int maxentry = current_cpu_data.tlbsize; | |
312 | ||
858dd5d4 SL |
313 | local_irq_save(flags); |
314 | ||
315 | old_entryhi = read_c0_entryhi(); | |
316 | old_pagemask = read_c0_pagemask(); | |
317 | ||
318 | /* Blast 'em all away. */ | |
319 | for (entry = 0; entry < maxentry; entry++) { | |
858dd5d4 | 320 | write_c0_index(entry); |
858dd5d4 SL |
321 | |
322 | if (skip_kseg0) { | |
138f7ad9 | 323 | mtc0_tlbr_hazard(); |
858dd5d4 | 324 | tlb_read(); |
138f7ad9 | 325 | tlb_read_hazard(); |
858dd5d4 SL |
326 | |
327 | entryhi = read_c0_entryhi(); | |
328 | ||
329 | /* Don't blow away guest kernel entries */ | |
d116e812 | 330 | if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0) |
858dd5d4 | 331 | continue; |
858dd5d4 SL |
332 | } |
333 | ||
334 | /* Make sure all entries differ. */ | |
335 | write_c0_entryhi(UNIQUE_ENTRYHI(entry)); | |
858dd5d4 | 336 | write_c0_entrylo0(0); |
858dd5d4 SL |
337 | write_c0_entrylo1(0); |
338 | mtc0_tlbw_hazard(); | |
339 | ||
340 | tlb_write_indexed(); | |
138f7ad9 | 341 | tlbw_use_hazard(); |
858dd5d4 SL |
342 | } |
343 | ||
858dd5d4 SL |
344 | write_c0_entryhi(old_entryhi); |
345 | write_c0_pagemask(old_pagemask); | |
346 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
347 | |
348 | local_irq_restore(flags); | |
349 | } | |
cb1b447f | 350 | EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb); |
858dd5d4 | 351 | |
858dd5d4 SL |
352 | void kvm_local_flush_tlb_all(void) |
353 | { | |
354 | unsigned long flags; | |
355 | unsigned long old_ctx; | |
356 | int entry = 0; | |
357 | ||
358 | local_irq_save(flags); | |
359 | /* Save old context and create impossible VPN2 value */ | |
360 | old_ctx = read_c0_entryhi(); | |
361 | write_c0_entrylo0(0); | |
362 | write_c0_entrylo1(0); | |
363 | ||
364 | /* Blast 'em all away. */ | |
365 | while (entry < current_cpu_data.tlbsize) { | |
366 | /* Make sure all entries differ. */ | |
367 | write_c0_entryhi(UNIQUE_ENTRYHI(entry)); | |
368 | write_c0_index(entry); | |
369 | mtc0_tlbw_hazard(); | |
370 | tlb_write_indexed(); | |
138f7ad9 | 371 | tlbw_use_hazard(); |
858dd5d4 SL |
372 | entry++; |
373 | } | |
858dd5d4 SL |
374 | write_c0_entryhi(old_ctx); |
375 | mtc0_tlbw_hazard(); | |
376 | ||
377 | local_irq_restore(flags); | |
378 | } | |
cb1b447f | 379 | EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all); |