Commit | Line | Data |
---|---|---|
858dd5d4 | 1 | /* |
d116e812 DCZ |
2 | * This file is subject to the terms and conditions of the GNU General Public |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that | |
7 | * TLB handlers run from KSEG0 | |
8 | * | |
9 | * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. | |
10 | * Authors: Sanjay Lal <sanjayl@kymasys.com> | |
11 | */ | |
858dd5d4 | 12 | |
858dd5d4 SL |
13 | #include <linux/sched.h> |
14 | #include <linux/smp.h> | |
15 | #include <linux/mm.h> | |
16 | #include <linux/delay.h> | |
403015b3 | 17 | #include <linux/export.h> |
858dd5d4 | 18 | #include <linux/kvm_host.h> |
6d17c0d1 SL |
19 | #include <linux/srcu.h> |
20 | ||
858dd5d4 SL |
21 | #include <asm/cpu.h> |
22 | #include <asm/bootinfo.h> | |
23 | #include <asm/mmu_context.h> | |
24 | #include <asm/pgtable.h> | |
25 | #include <asm/cacheflush.h> | |
e36059e5 | 26 | #include <asm/tlb.h> |
e922a4cb | 27 | #include <asm/tlbdebug.h> |
858dd5d4 SL |
28 | |
29 | #undef CONFIG_MIPS_MT | |
30 | #include <asm/r4kcache.h> | |
31 | #define CONFIG_MIPS_MT | |
32 | ||
33 | #define KVM_GUEST_PC_TLB 0 | |
34 | #define KVM_GUEST_SP_TLB 1 | |
35 | ||
858dd5d4 | 36 | atomic_t kvm_mips_instance; |
cb1b447f | 37 | EXPORT_SYMBOL_GPL(kvm_mips_instance); |
858dd5d4 | 38 | |
403015b3 | 39 | static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu) |
858dd5d4 | 40 | { |
4edf00a4 PB |
41 | int cpu = smp_processor_id(); |
42 | ||
43 | return vcpu->arch.guest_kernel_asid[cpu] & | |
44 | cpu_asid_mask(&cpu_data[cpu]); | |
858dd5d4 SL |
45 | } |
46 | ||
403015b3 | 47 | static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu) |
858dd5d4 | 48 | { |
4edf00a4 PB |
49 | int cpu = smp_processor_id(); |
50 | ||
51 | return vcpu->arch.guest_user_asid[cpu] & | |
52 | cpu_asid_mask(&cpu_data[cpu]); | |
858dd5d4 SL |
53 | } |
54 | ||
bdb7ed86 | 55 | inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu) |
858dd5d4 SL |
56 | { |
57 | return vcpu->kvm->arch.commpage_tlb; | |
58 | } | |
59 | ||
d116e812 | 60 | /* Structure defining an tlb entry data set. */ |
858dd5d4 SL |
61 | |
62 | void kvm_mips_dump_host_tlbs(void) | |
63 | { | |
858dd5d4 | 64 | unsigned long flags; |
858dd5d4 SL |
65 | |
66 | local_irq_save(flags); | |
67 | ||
6ad78a5c | 68 | kvm_info("HOST TLBs:\n"); |
e922a4cb JH |
69 | dump_tlb_regs(); |
70 | pr_info("\n"); | |
71 | dump_tlb_all(); | |
72 | ||
858dd5d4 SL |
73 | local_irq_restore(flags); |
74 | } | |
cb1b447f | 75 | EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs); |
858dd5d4 SL |
76 | |
77 | void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu) | |
78 | { | |
79 | struct mips_coproc *cop0 = vcpu->arch.cop0; | |
80 | struct kvm_mips_tlb tlb; | |
81 | int i; | |
82 | ||
6ad78a5c DCZ |
83 | kvm_info("Guest TLBs:\n"); |
84 | kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0)); | |
858dd5d4 SL |
85 | |
86 | for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { | |
87 | tlb = vcpu->arch.guest_tlb[i]; | |
6ad78a5c | 88 | kvm_info("TLB%c%3d Hi 0x%08lx ", |
9fbfb06a JH |
89 | (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & MIPS3_PG_V |
90 | ? ' ' : '*', | |
6ad78a5c | 91 | i, tlb.tlb_hi); |
8cffd197 | 92 | kvm_info("Lo0=0x%09llx %c%c attr %lx ", |
9fbfb06a JH |
93 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]), |
94 | (tlb.tlb_lo[0] & MIPS3_PG_D) ? 'D' : ' ', | |
95 | (tlb.tlb_lo[0] & MIPS3_PG_G) ? 'G' : ' ', | |
96 | (tlb.tlb_lo[0] >> 3) & 7); | |
8cffd197 | 97 | kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n", |
9fbfb06a JH |
98 | (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]), |
99 | (tlb.tlb_lo[1] & MIPS3_PG_D) ? 'D' : ' ', | |
100 | (tlb.tlb_lo[1] & MIPS3_PG_G) ? 'G' : ' ', | |
101 | (tlb.tlb_lo[1] >> 3) & 7, tlb.tlb_mask); | |
858dd5d4 SL |
102 | } |
103 | } | |
cb1b447f | 104 | EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs); |
858dd5d4 | 105 | |
858dd5d4 SL |
106 | /* XXXKYMA: Must be called with interrupts disabled */ |
107 | /* set flush_dcache_mask == 0 if no dcache flush required */ | |
d116e812 DCZ |
108 | int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi, |
109 | unsigned long entrylo0, unsigned long entrylo1, | |
110 | int flush_dcache_mask) | |
858dd5d4 SL |
111 | { |
112 | unsigned long flags; | |
113 | unsigned long old_entryhi; | |
b045c406 | 114 | int idx; |
858dd5d4 SL |
115 | |
116 | local_irq_save(flags); | |
117 | ||
858dd5d4 SL |
118 | old_entryhi = read_c0_entryhi(); |
119 | write_c0_entryhi(entryhi); | |
120 | mtc0_tlbw_hazard(); | |
121 | ||
122 | tlb_probe(); | |
123 | tlb_probe_hazard(); | |
124 | idx = read_c0_index(); | |
125 | ||
126 | if (idx > current_cpu_data.tlbsize) { | |
127 | kvm_err("%s: Invalid Index: %d\n", __func__, idx); | |
128 | kvm_mips_dump_host_tlbs(); | |
cfec0e75 | 129 | local_irq_restore(flags); |
858dd5d4 SL |
130 | return -1; |
131 | } | |
132 | ||
858dd5d4 SL |
133 | write_c0_entrylo0(entrylo0); |
134 | write_c0_entrylo1(entrylo1); | |
135 | mtc0_tlbw_hazard(); | |
136 | ||
b5dfc6c1 JH |
137 | if (idx < 0) |
138 | tlb_write_random(); | |
139 | else | |
140 | tlb_write_indexed(); | |
858dd5d4 SL |
141 | tlbw_use_hazard(); |
142 | ||
3d654833 JH |
143 | kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n", |
144 | vcpu->arch.pc, idx, read_c0_entryhi(), | |
145 | read_c0_entrylo0(), read_c0_entrylo1()); | |
858dd5d4 SL |
146 | |
147 | /* Flush D-cache */ | |
148 | if (flush_dcache_mask) { | |
149 | if (entrylo0 & MIPS3_PG_V) { | |
150 | ++vcpu->stat.flush_dcache_exits; | |
d116e812 DCZ |
151 | flush_data_cache_page((entryhi & VPN2_MASK) & |
152 | ~flush_dcache_mask); | |
858dd5d4 SL |
153 | } |
154 | if (entrylo1 & MIPS3_PG_V) { | |
155 | ++vcpu->stat.flush_dcache_exits; | |
d116e812 DCZ |
156 | flush_data_cache_page(((entryhi & VPN2_MASK) & |
157 | ~flush_dcache_mask) | | |
158 | (0x1 << PAGE_SHIFT)); | |
858dd5d4 SL |
159 | } |
160 | } | |
161 | ||
162 | /* Restore old ASID */ | |
163 | write_c0_entryhi(old_entryhi); | |
164 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
165 | local_irq_restore(flags); |
166 | return 0; | |
167 | } | |
403015b3 | 168 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write); |
858dd5d4 SL |
169 | |
170 | int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr, | |
171 | struct kvm_vcpu *vcpu) | |
172 | { | |
ba049e93 | 173 | kvm_pfn_t pfn0, pfn1; |
858dd5d4 SL |
174 | unsigned long flags, old_entryhi = 0, vaddr = 0; |
175 | unsigned long entrylo0 = 0, entrylo1 = 0; | |
176 | ||
858dd5d4 SL |
177 | pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT; |
178 | pfn1 = 0; | |
d116e812 DCZ |
179 | entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | |
180 | (1 << 2) | (0x1 << 1); | |
858dd5d4 SL |
181 | entrylo1 = 0; |
182 | ||
183 | local_irq_save(flags); | |
184 | ||
185 | old_entryhi = read_c0_entryhi(); | |
186 | vaddr = badvaddr & (PAGE_MASK << 1); | |
187 | write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu)); | |
858dd5d4 | 188 | write_c0_entrylo0(entrylo0); |
858dd5d4 | 189 | write_c0_entrylo1(entrylo1); |
858dd5d4 SL |
190 | write_c0_index(kvm_mips_get_commpage_asid(vcpu)); |
191 | mtc0_tlbw_hazard(); | |
192 | tlb_write_indexed(); | |
858dd5d4 SL |
193 | tlbw_use_hazard(); |
194 | ||
d116e812 DCZ |
195 | kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n", |
196 | vcpu->arch.pc, read_c0_index(), read_c0_entryhi(), | |
197 | read_c0_entrylo0(), read_c0_entrylo1()); | |
858dd5d4 SL |
198 | |
199 | /* Restore old ASID */ | |
200 | write_c0_entryhi(old_entryhi); | |
201 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
202 | local_irq_restore(flags); |
203 | ||
204 | return 0; | |
205 | } | |
cb1b447f | 206 | EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault); |
858dd5d4 | 207 | |
858dd5d4 SL |
208 | int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi) |
209 | { | |
210 | int i; | |
211 | int index = -1; | |
212 | struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb; | |
213 | ||
858dd5d4 | 214 | for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { |
d116e812 DCZ |
215 | if (TLB_HI_VPN2_HIT(tlb[i], entryhi) && |
216 | TLB_HI_ASID_HIT(tlb[i], entryhi)) { | |
858dd5d4 SL |
217 | index = i; |
218 | break; | |
219 | } | |
220 | } | |
221 | ||
858dd5d4 | 222 | kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n", |
9fbfb06a | 223 | __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]); |
858dd5d4 SL |
224 | |
225 | return index; | |
226 | } | |
cb1b447f | 227 | EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup); |
858dd5d4 SL |
228 | |
229 | int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr) | |
230 | { | |
231 | unsigned long old_entryhi, flags; | |
b045c406 | 232 | int idx; |
858dd5d4 | 233 | |
858dd5d4 SL |
234 | local_irq_save(flags); |
235 | ||
236 | old_entryhi = read_c0_entryhi(); | |
237 | ||
238 | if (KVM_GUEST_KERNEL_MODE(vcpu)) | |
d116e812 DCZ |
239 | write_c0_entryhi((vaddr & VPN2_MASK) | |
240 | kvm_mips_get_kernel_asid(vcpu)); | |
858dd5d4 | 241 | else { |
d116e812 DCZ |
242 | write_c0_entryhi((vaddr & VPN2_MASK) | |
243 | kvm_mips_get_user_asid(vcpu)); | |
858dd5d4 SL |
244 | } |
245 | ||
246 | mtc0_tlbw_hazard(); | |
247 | ||
248 | tlb_probe(); | |
249 | tlb_probe_hazard(); | |
250 | idx = read_c0_index(); | |
251 | ||
252 | /* Restore old ASID */ | |
253 | write_c0_entryhi(old_entryhi); | |
254 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
255 | |
256 | local_irq_restore(flags); | |
257 | ||
858dd5d4 | 258 | kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx); |
858dd5d4 SL |
259 | |
260 | return idx; | |
261 | } | |
cb1b447f | 262 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup); |
858dd5d4 SL |
263 | |
264 | int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va) | |
265 | { | |
266 | int idx; | |
267 | unsigned long flags, old_entryhi; | |
268 | ||
269 | local_irq_save(flags); | |
270 | ||
858dd5d4 SL |
271 | old_entryhi = read_c0_entryhi(); |
272 | ||
273 | write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu)); | |
274 | mtc0_tlbw_hazard(); | |
275 | ||
276 | tlb_probe(); | |
277 | tlb_probe_hazard(); | |
278 | idx = read_c0_index(); | |
279 | ||
280 | if (idx >= current_cpu_data.tlbsize) | |
281 | BUG(); | |
282 | ||
283 | if (idx > 0) { | |
284 | write_c0_entryhi(UNIQUE_ENTRYHI(idx)); | |
858dd5d4 | 285 | write_c0_entrylo0(0); |
858dd5d4 SL |
286 | write_c0_entrylo1(0); |
287 | mtc0_tlbw_hazard(); | |
288 | ||
289 | tlb_write_indexed(); | |
138f7ad9 | 290 | tlbw_use_hazard(); |
858dd5d4 SL |
291 | } |
292 | ||
293 | write_c0_entryhi(old_entryhi); | |
294 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
295 | |
296 | local_irq_restore(flags); | |
297 | ||
3d654833 | 298 | if (idx > 0) |
858dd5d4 | 299 | kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__, |
3d654833 | 300 | (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx); |
858dd5d4 SL |
301 | |
302 | return 0; | |
303 | } | |
cb1b447f | 304 | EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv); |
858dd5d4 SL |
305 | |
306 | void kvm_mips_flush_host_tlb(int skip_kseg0) | |
307 | { | |
308 | unsigned long flags; | |
309 | unsigned long old_entryhi, entryhi; | |
310 | unsigned long old_pagemask; | |
311 | int entry = 0; | |
312 | int maxentry = current_cpu_data.tlbsize; | |
313 | ||
858dd5d4 SL |
314 | local_irq_save(flags); |
315 | ||
316 | old_entryhi = read_c0_entryhi(); | |
317 | old_pagemask = read_c0_pagemask(); | |
318 | ||
319 | /* Blast 'em all away. */ | |
320 | for (entry = 0; entry < maxentry; entry++) { | |
858dd5d4 | 321 | write_c0_index(entry); |
858dd5d4 SL |
322 | |
323 | if (skip_kseg0) { | |
138f7ad9 | 324 | mtc0_tlbr_hazard(); |
858dd5d4 | 325 | tlb_read(); |
138f7ad9 | 326 | tlb_read_hazard(); |
858dd5d4 SL |
327 | |
328 | entryhi = read_c0_entryhi(); | |
329 | ||
330 | /* Don't blow away guest kernel entries */ | |
d116e812 | 331 | if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0) |
858dd5d4 | 332 | continue; |
858dd5d4 SL |
333 | } |
334 | ||
335 | /* Make sure all entries differ. */ | |
336 | write_c0_entryhi(UNIQUE_ENTRYHI(entry)); | |
858dd5d4 | 337 | write_c0_entrylo0(0); |
858dd5d4 SL |
338 | write_c0_entrylo1(0); |
339 | mtc0_tlbw_hazard(); | |
340 | ||
341 | tlb_write_indexed(); | |
138f7ad9 | 342 | tlbw_use_hazard(); |
858dd5d4 SL |
343 | } |
344 | ||
858dd5d4 SL |
345 | write_c0_entryhi(old_entryhi); |
346 | write_c0_pagemask(old_pagemask); | |
347 | mtc0_tlbw_hazard(); | |
858dd5d4 SL |
348 | |
349 | local_irq_restore(flags); | |
350 | } | |
cb1b447f | 351 | EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb); |
858dd5d4 | 352 | |
858dd5d4 SL |
353 | void kvm_local_flush_tlb_all(void) |
354 | { | |
355 | unsigned long flags; | |
356 | unsigned long old_ctx; | |
357 | int entry = 0; | |
358 | ||
359 | local_irq_save(flags); | |
360 | /* Save old context and create impossible VPN2 value */ | |
361 | old_ctx = read_c0_entryhi(); | |
362 | write_c0_entrylo0(0); | |
363 | write_c0_entrylo1(0); | |
364 | ||
365 | /* Blast 'em all away. */ | |
366 | while (entry < current_cpu_data.tlbsize) { | |
367 | /* Make sure all entries differ. */ | |
368 | write_c0_entryhi(UNIQUE_ENTRYHI(entry)); | |
369 | write_c0_index(entry); | |
370 | mtc0_tlbw_hazard(); | |
371 | tlb_write_indexed(); | |
138f7ad9 | 372 | tlbw_use_hazard(); |
858dd5d4 SL |
373 | entry++; |
374 | } | |
858dd5d4 SL |
375 | write_c0_entryhi(old_ctx); |
376 | mtc0_tlbw_hazard(); | |
377 | ||
378 | local_irq_restore(flags); | |
379 | } | |
cb1b447f | 380 | EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all); |