2 * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved.
4 * Author: Yu Liu, yu.liu@freescale.com
7 * This file is based on arch/powerpc/kvm/44x_tlb.c,
8 * by Hollis Blanchard <hollisb@us.ibm.com>.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License, version 2, as
12 * published by the Free Software Foundation.
15 #include <linux/kernel.h>
16 #include <linux/types.h>
17 #include <linux/slab.h>
18 #include <linux/string.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_host.h>
21 #include <linux/highmem.h>
22 #include <linux/log2.h>
23 #include <linux/uaccess.h>
24 #include <linux/sched.h>
25 #include <linux/rwsem.h>
26 #include <linux/vmalloc.h>
27 #include <linux/hugetlb.h>
28 #include <asm/kvm_ppc.h>
34 #define to_htlb1_esel(esel) (host_tlb_params[1].entries - (esel) - 1)
36 static struct kvmppc_e500_tlb_params host_tlb_params
[E500_TLB_NUM
];
38 static inline unsigned int gtlb0_get_next_victim(
39 struct kvmppc_vcpu_e500
*vcpu_e500
)
43 victim
= vcpu_e500
->gtlb_nv
[0]++;
44 if (unlikely(vcpu_e500
->gtlb_nv
[0] >= vcpu_e500
->gtlb_params
[0].ways
))
45 vcpu_e500
->gtlb_nv
[0] = 0;
50 static inline unsigned int tlb1_max_shadow_size(void)
52 /* reserve one entry for magic page */
53 return host_tlb_params
[1].entries
- tlbcam_index
- 1;
56 static inline int tlbe_is_writable(struct kvm_book3e_206_tlb_entry
*tlbe
)
58 return tlbe
->mas7_3
& (MAS3_SW
|MAS3_UW
);
61 static inline u32
e500_shadow_mas3_attrib(u32 mas3
, int usermode
)
63 /* Mask off reserved bits. */
64 mas3
&= MAS3_ATTRIB_MASK
;
67 /* Guest is in supervisor mode,
68 * so we need to translate guest
69 * supervisor permissions into user permissions. */
70 mas3
&= ~E500_TLB_USER_PERM_MASK
;
71 mas3
|= (mas3
& E500_TLB_SUPER_PERM_MASK
) << 1;
74 return mas3
| E500_TLB_SUPER_PERM_MASK
;
77 static inline u32
e500_shadow_mas2_attrib(u32 mas2
, int usermode
)
80 return (mas2
& MAS2_ATTRIB_MASK
) | MAS2_M
;
82 return mas2
& MAS2_ATTRIB_MASK
;
87 * writing shadow tlb entry to host TLB
89 static inline void __write_host_tlbe(struct kvm_book3e_206_tlb_entry
*stlbe
,
94 local_irq_save(flags
);
95 mtspr(SPRN_MAS0
, mas0
);
96 mtspr(SPRN_MAS1
, stlbe
->mas1
);
97 mtspr(SPRN_MAS2
, (unsigned long)stlbe
->mas2
);
98 mtspr(SPRN_MAS3
, (u32
)stlbe
->mas7_3
);
99 mtspr(SPRN_MAS7
, (u32
)(stlbe
->mas7_3
>> 32));
100 asm volatile("isync; tlbwe" : : : "memory");
101 local_irq_restore(flags
);
103 trace_kvm_booke206_stlb_write(mas0
, stlbe
->mas8
, stlbe
->mas1
,
104 stlbe
->mas2
, stlbe
->mas7_3
);
108 * Acquire a mas0 with victim hint, as if we just took a TLB miss.
110 * We don't care about the address we're searching for, other than that it's
111 * in the right set and is not present in the TLB. Using a zero PID and a
112 * userspace address means we don't have to set and then restore MAS5, or
113 * calculate a proper MAS6 value.
115 static u32
get_host_mas0(unsigned long eaddr
)
120 local_irq_save(flags
);
122 asm volatile("tlbsx 0, %0" : : "b" (eaddr
& ~CONFIG_PAGE_OFFSET
));
123 mas0
= mfspr(SPRN_MAS0
);
124 local_irq_restore(flags
);
129 /* sesel is for tlb1 only */
130 static inline void write_host_tlbe(struct kvmppc_vcpu_e500
*vcpu_e500
,
131 int tlbsel
, int sesel
, struct kvm_book3e_206_tlb_entry
*stlbe
)
136 mas0
= get_host_mas0(stlbe
->mas2
);
137 __write_host_tlbe(stlbe
, mas0
);
139 __write_host_tlbe(stlbe
,
141 MAS0_ESEL(to_htlb1_esel(sesel
)));
145 #ifdef CONFIG_KVM_E500
146 void kvmppc_map_magic(struct kvm_vcpu
*vcpu
)
148 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
149 struct kvm_book3e_206_tlb_entry magic
;
150 ulong shared_page
= ((ulong
)vcpu
->arch
.shared
) & PAGE_MASK
;
154 pfn
= (pfn_t
)virt_to_phys((void *)shared_page
) >> PAGE_SHIFT
;
155 get_page(pfn_to_page(pfn
));
158 stid
= kvmppc_e500_get_sid(vcpu_e500
, 0, 0, 0, 0);
160 magic
.mas1
= MAS1_VALID
| MAS1_TS
| MAS1_TID(stid
) |
161 MAS1_TSIZE(BOOK3E_PAGESZ_4K
);
162 magic
.mas2
= vcpu
->arch
.magic_page_ea
| MAS2_M
;
163 magic
.mas7_3
= ((u64
)pfn
<< PAGE_SHIFT
) |
164 MAS3_SW
| MAS3_SR
| MAS3_UW
| MAS3_UR
;
167 __write_host_tlbe(&magic
, MAS0_TLBSEL(1) | MAS0_ESEL(tlbcam_index
));
172 static void inval_gtlbe_on_host(struct kvmppc_vcpu_e500
*vcpu_e500
,
173 int tlbsel
, int esel
)
175 struct kvm_book3e_206_tlb_entry
*gtlbe
=
176 get_entry(vcpu_e500
, tlbsel
, esel
);
179 kvmppc_e500_tlbil_all(vcpu_e500
);
183 /* Guest tlbe is backed by at most one host tlbe per shadow pid. */
184 kvmppc_e500_tlbil_one(vcpu_e500
, gtlbe
);
187 static int tlb0_set_base(gva_t addr
, int sets
, int ways
)
191 set_base
= (addr
>> PAGE_SHIFT
) & (sets
- 1);
197 static int gtlb0_set_base(struct kvmppc_vcpu_e500
*vcpu_e500
, gva_t addr
)
199 return tlb0_set_base(addr
, vcpu_e500
->gtlb_params
[0].sets
,
200 vcpu_e500
->gtlb_params
[0].ways
);
203 static unsigned int get_tlb_esel(struct kvm_vcpu
*vcpu
, int tlbsel
)
205 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
206 int esel
= get_tlb_esel_bit(vcpu
);
209 esel
&= vcpu_e500
->gtlb_params
[0].ways
- 1;
210 esel
+= gtlb0_set_base(vcpu_e500
, vcpu
->arch
.shared
->mas2
);
212 esel
&= vcpu_e500
->gtlb_params
[tlbsel
].entries
- 1;
218 /* Search the guest TLB for a matching entry. */
219 static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500
*vcpu_e500
,
220 gva_t eaddr
, int tlbsel
, unsigned int pid
, int as
)
222 int size
= vcpu_e500
->gtlb_params
[tlbsel
].entries
;
223 unsigned int set_base
, offset
;
227 set_base
= gtlb0_set_base(vcpu_e500
, eaddr
);
228 size
= vcpu_e500
->gtlb_params
[0].ways
;
233 offset
= vcpu_e500
->gtlb_offset
[tlbsel
];
235 for (i
= 0; i
< size
; i
++) {
236 struct kvm_book3e_206_tlb_entry
*tlbe
=
237 &vcpu_e500
->gtlb_arch
[offset
+ set_base
+ i
];
240 if (eaddr
< get_tlb_eaddr(tlbe
))
243 if (eaddr
> get_tlb_end(tlbe
))
246 tid
= get_tlb_tid(tlbe
);
247 if (tid
&& (tid
!= pid
))
250 if (!get_tlb_v(tlbe
))
253 if (get_tlb_ts(tlbe
) != as
&& as
!= -1)
262 static inline void kvmppc_e500_ref_setup(struct tlbe_ref
*ref
,
263 struct kvm_book3e_206_tlb_entry
*gtlbe
,
267 ref
->flags
= E500_TLB_VALID
;
269 if (tlbe_is_writable(gtlbe
))
270 ref
->flags
|= E500_TLB_DIRTY
;
273 static inline void kvmppc_e500_ref_release(struct tlbe_ref
*ref
)
275 if (ref
->flags
& E500_TLB_VALID
) {
276 if (ref
->flags
& E500_TLB_DIRTY
)
277 kvm_release_pfn_dirty(ref
->pfn
);
279 kvm_release_pfn_clean(ref
->pfn
);
285 static void clear_tlb_privs(struct kvmppc_vcpu_e500
*vcpu_e500
)
290 for (i
= 0; i
< vcpu_e500
->gtlb_params
[tlbsel
].entries
; i
++) {
291 struct tlbe_ref
*ref
=
292 &vcpu_e500
->gtlb_priv
[tlbsel
][i
].ref
;
293 kvmppc_e500_ref_release(ref
);
297 static void clear_tlb_refs(struct kvmppc_vcpu_e500
*vcpu_e500
)
302 kvmppc_e500_tlbil_all(vcpu_e500
);
304 for (i
= 0; i
< host_tlb_params
[stlbsel
].entries
; i
++) {
305 struct tlbe_ref
*ref
=
306 &vcpu_e500
->tlb_refs
[stlbsel
][i
];
307 kvmppc_e500_ref_release(ref
);
310 clear_tlb_privs(vcpu_e500
);
313 static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu
*vcpu
,
314 unsigned int eaddr
, int as
)
316 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
317 unsigned int victim
, tsized
;
320 /* since we only have two TLBs, only lower bit is used. */
321 tlbsel
= (vcpu
->arch
.shared
->mas4
>> 28) & 0x1;
322 victim
= (tlbsel
== 0) ? gtlb0_get_next_victim(vcpu_e500
) : 0;
323 tsized
= (vcpu
->arch
.shared
->mas4
>> 7) & 0x1f;
325 vcpu
->arch
.shared
->mas0
= MAS0_TLBSEL(tlbsel
) | MAS0_ESEL(victim
)
326 | MAS0_NV(vcpu_e500
->gtlb_nv
[tlbsel
]);
327 vcpu
->arch
.shared
->mas1
= MAS1_VALID
| (as
? MAS1_TS
: 0)
328 | MAS1_TID(get_tlbmiss_tid(vcpu
))
329 | MAS1_TSIZE(tsized
);
330 vcpu
->arch
.shared
->mas2
= (eaddr
& MAS2_EPN
)
331 | (vcpu
->arch
.shared
->mas4
& MAS2_ATTRIB_MASK
);
332 vcpu
->arch
.shared
->mas7_3
&= MAS3_U0
| MAS3_U1
| MAS3_U2
| MAS3_U3
;
333 vcpu
->arch
.shared
->mas6
= (vcpu
->arch
.shared
->mas6
& MAS6_SPID1
)
334 | (get_cur_pid(vcpu
) << 16)
335 | (as
? MAS6_SAS
: 0);
338 /* TID must be supplied by the caller */
339 static inline void kvmppc_e500_setup_stlbe(
340 struct kvm_vcpu
*vcpu
,
341 struct kvm_book3e_206_tlb_entry
*gtlbe
,
342 int tsize
, struct tlbe_ref
*ref
, u64 gvaddr
,
343 struct kvm_book3e_206_tlb_entry
*stlbe
)
345 pfn_t pfn
= ref
->pfn
;
346 u32 pr
= vcpu
->arch
.shared
->msr
& MSR_PR
;
348 BUG_ON(!(ref
->flags
& E500_TLB_VALID
));
350 /* Force IPROT=0 for all guest mappings. */
351 stlbe
->mas1
= MAS1_TSIZE(tsize
) | get_tlb_sts(gtlbe
) | MAS1_VALID
;
352 stlbe
->mas2
= (gvaddr
& MAS2_EPN
) |
353 e500_shadow_mas2_attrib(gtlbe
->mas2
, pr
);
354 stlbe
->mas7_3
= ((u64
)pfn
<< PAGE_SHIFT
) |
355 e500_shadow_mas3_attrib(gtlbe
->mas7_3
, pr
);
358 static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500
*vcpu_e500
,
359 u64 gvaddr
, gfn_t gfn
, struct kvm_book3e_206_tlb_entry
*gtlbe
,
360 int tlbsel
, struct kvm_book3e_206_tlb_entry
*stlbe
,
361 struct tlbe_ref
*ref
)
363 struct kvm_memory_slot
*slot
;
364 unsigned long pfn
, hva
;
366 int tsize
= BOOK3E_PAGESZ_4K
;
369 * Translate guest physical to true physical, acquiring
370 * a page reference if it is normal, non-reserved memory.
372 * gfn_to_memslot() must succeed because otherwise we wouldn't
373 * have gotten this far. Eventually we should just pass the slot
374 * pointer through from the first lookup.
376 slot
= gfn_to_memslot(vcpu_e500
->vcpu
.kvm
, gfn
);
377 hva
= gfn_to_hva_memslot(slot
, gfn
);
380 struct vm_area_struct
*vma
;
381 down_read(¤t
->mm
->mmap_sem
);
383 vma
= find_vma(current
->mm
, hva
);
384 if (vma
&& hva
>= vma
->vm_start
&&
385 (vma
->vm_flags
& VM_PFNMAP
)) {
387 * This VMA is a physically contiguous region (e.g.
388 * /dev/mem) that bypasses normal Linux page
389 * management. Find the overlap between the
390 * vma and the memslot.
393 unsigned long start
, end
;
394 unsigned long slot_start
, slot_end
;
398 start
= vma
->vm_pgoff
;
400 ((vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
);
402 pfn
= start
+ ((hva
- vma
->vm_start
) >> PAGE_SHIFT
);
404 slot_start
= pfn
- (gfn
- slot
->base_gfn
);
405 slot_end
= slot_start
+ slot
->npages
;
407 if (start
< slot_start
)
412 tsize
= (gtlbe
->mas1
& MAS1_TSIZE_MASK
) >>
416 * e500 doesn't implement the lowest tsize bit,
419 tsize
= max(BOOK3E_PAGESZ_4K
, tsize
& ~1);
422 * Now find the largest tsize (up to what the guest
423 * requested) that will cover gfn, stay within the
424 * range, and for which gfn and pfn are mutually
428 for (; tsize
> BOOK3E_PAGESZ_4K
; tsize
-= 2) {
429 unsigned long gfn_start
, gfn_end
, tsize_pages
;
430 tsize_pages
= 1 << (tsize
- 2);
432 gfn_start
= gfn
& ~(tsize_pages
- 1);
433 gfn_end
= gfn_start
+ tsize_pages
;
435 if (gfn_start
+ pfn
- gfn
< start
)
437 if (gfn_end
+ pfn
- gfn
> end
)
439 if ((gfn
& (tsize_pages
- 1)) !=
440 (pfn
& (tsize_pages
- 1)))
443 gvaddr
&= ~((tsize_pages
<< PAGE_SHIFT
) - 1);
444 pfn
&= ~(tsize_pages
- 1);
447 } else if (vma
&& hva
>= vma
->vm_start
&&
448 (vma
->vm_flags
& VM_HUGETLB
)) {
449 unsigned long psize
= vma_kernel_pagesize(vma
);
451 tsize
= (gtlbe
->mas1
& MAS1_TSIZE_MASK
) >>
455 * Take the largest page size that satisfies both host
458 tsize
= min(__ilog2(psize
) - 10, tsize
);
461 * e500 doesn't implement the lowest tsize bit,
464 tsize
= max(BOOK3E_PAGESZ_4K
, tsize
& ~1);
467 up_read(¤t
->mm
->mmap_sem
);
470 if (likely(!pfnmap
)) {
471 unsigned long tsize_pages
= 1 << (tsize
+ 10 - PAGE_SHIFT
);
472 pfn
= gfn_to_pfn_memslot(vcpu_e500
->vcpu
.kvm
, slot
, gfn
);
473 if (is_error_pfn(pfn
)) {
474 printk(KERN_ERR
"Couldn't get real page for gfn %lx!\n",
476 kvm_release_pfn_clean(pfn
);
480 /* Align guest and physical address to page map boundaries */
481 pfn
&= ~(tsize_pages
- 1);
482 gvaddr
&= ~((tsize_pages
<< PAGE_SHIFT
) - 1);
485 /* Drop old ref and setup new one. */
486 kvmppc_e500_ref_release(ref
);
487 kvmppc_e500_ref_setup(ref
, gtlbe
, pfn
);
489 kvmppc_e500_setup_stlbe(&vcpu_e500
->vcpu
, gtlbe
, tsize
,
493 /* XXX only map the one-one case, for now use TLB0 */
494 static void kvmppc_e500_tlb0_map(struct kvmppc_vcpu_e500
*vcpu_e500
,
496 struct kvm_book3e_206_tlb_entry
*stlbe
)
498 struct kvm_book3e_206_tlb_entry
*gtlbe
;
499 struct tlbe_ref
*ref
;
501 gtlbe
= get_entry(vcpu_e500
, 0, esel
);
502 ref
= &vcpu_e500
->gtlb_priv
[0][esel
].ref
;
504 kvmppc_e500_shadow_map(vcpu_e500
, get_tlb_eaddr(gtlbe
),
505 get_tlb_raddr(gtlbe
) >> PAGE_SHIFT
,
506 gtlbe
, 0, stlbe
, ref
);
509 /* Caller must ensure that the specified guest TLB entry is safe to insert into
511 /* XXX for both one-one and one-to-many , for now use TLB1 */
512 static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500
*vcpu_e500
,
513 u64 gvaddr
, gfn_t gfn
, struct kvm_book3e_206_tlb_entry
*gtlbe
,
514 struct kvm_book3e_206_tlb_entry
*stlbe
)
516 struct tlbe_ref
*ref
;
519 victim
= vcpu_e500
->host_tlb1_nv
++;
521 if (unlikely(vcpu_e500
->host_tlb1_nv
>= tlb1_max_shadow_size()))
522 vcpu_e500
->host_tlb1_nv
= 0;
524 ref
= &vcpu_e500
->tlb_refs
[1][victim
];
525 kvmppc_e500_shadow_map(vcpu_e500
, gvaddr
, gfn
, gtlbe
, 1, stlbe
, ref
);
530 static inline int kvmppc_e500_gtlbe_invalidate(
531 struct kvmppc_vcpu_e500
*vcpu_e500
,
532 int tlbsel
, int esel
)
534 struct kvm_book3e_206_tlb_entry
*gtlbe
=
535 get_entry(vcpu_e500
, tlbsel
, esel
);
537 if (unlikely(get_tlb_iprot(gtlbe
)))
545 int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500
*vcpu_e500
, ulong value
)
549 if (value
& MMUCSR0_TLB0FI
)
550 for (esel
= 0; esel
< vcpu_e500
->gtlb_params
[0].entries
; esel
++)
551 kvmppc_e500_gtlbe_invalidate(vcpu_e500
, 0, esel
);
552 if (value
& MMUCSR0_TLB1FI
)
553 for (esel
= 0; esel
< vcpu_e500
->gtlb_params
[1].entries
; esel
++)
554 kvmppc_e500_gtlbe_invalidate(vcpu_e500
, 1, esel
);
556 /* Invalidate all vcpu id mappings */
557 kvmppc_e500_tlbil_all(vcpu_e500
);
562 int kvmppc_e500_emul_tlbivax(struct kvm_vcpu
*vcpu
, int ra
, int rb
)
564 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
569 ea
= ((ra
) ? kvmppc_get_gpr(vcpu
, ra
) : 0) + kvmppc_get_gpr(vcpu
, rb
);
571 ia
= (ea
>> 2) & 0x1;
573 /* since we only have two TLBs, only lower bit is used. */
574 tlbsel
= (ea
>> 3) & 0x1;
577 /* invalidate all entries */
578 for (esel
= 0; esel
< vcpu_e500
->gtlb_params
[tlbsel
].entries
;
580 kvmppc_e500_gtlbe_invalidate(vcpu_e500
, tlbsel
, esel
);
583 esel
= kvmppc_e500_tlb_index(vcpu_e500
, ea
, tlbsel
,
584 get_cur_pid(vcpu
), -1);
586 kvmppc_e500_gtlbe_invalidate(vcpu_e500
, tlbsel
, esel
);
589 /* Invalidate all vcpu id mappings */
590 kvmppc_e500_tlbil_all(vcpu_e500
);
595 int kvmppc_e500_emul_tlbre(struct kvm_vcpu
*vcpu
)
597 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
599 struct kvm_book3e_206_tlb_entry
*gtlbe
;
601 tlbsel
= get_tlb_tlbsel(vcpu
);
602 esel
= get_tlb_esel(vcpu
, tlbsel
);
604 gtlbe
= get_entry(vcpu_e500
, tlbsel
, esel
);
605 vcpu
->arch
.shared
->mas0
&= ~MAS0_NV(~0);
606 vcpu
->arch
.shared
->mas0
|= MAS0_NV(vcpu_e500
->gtlb_nv
[tlbsel
]);
607 vcpu
->arch
.shared
->mas1
= gtlbe
->mas1
;
608 vcpu
->arch
.shared
->mas2
= gtlbe
->mas2
;
609 vcpu
->arch
.shared
->mas7_3
= gtlbe
->mas7_3
;
614 int kvmppc_e500_emul_tlbsx(struct kvm_vcpu
*vcpu
, int rb
)
616 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
617 int as
= !!get_cur_sas(vcpu
);
618 unsigned int pid
= get_cur_spid(vcpu
);
620 struct kvm_book3e_206_tlb_entry
*gtlbe
= NULL
;
623 ea
= kvmppc_get_gpr(vcpu
, rb
);
625 for (tlbsel
= 0; tlbsel
< 2; tlbsel
++) {
626 esel
= kvmppc_e500_tlb_index(vcpu_e500
, ea
, tlbsel
, pid
, as
);
628 gtlbe
= get_entry(vcpu_e500
, tlbsel
, esel
);
634 esel
&= vcpu_e500
->gtlb_params
[tlbsel
].ways
- 1;
636 vcpu
->arch
.shared
->mas0
= MAS0_TLBSEL(tlbsel
) | MAS0_ESEL(esel
)
637 | MAS0_NV(vcpu_e500
->gtlb_nv
[tlbsel
]);
638 vcpu
->arch
.shared
->mas1
= gtlbe
->mas1
;
639 vcpu
->arch
.shared
->mas2
= gtlbe
->mas2
;
640 vcpu
->arch
.shared
->mas7_3
= gtlbe
->mas7_3
;
644 /* since we only have two TLBs, only lower bit is used. */
645 tlbsel
= vcpu
->arch
.shared
->mas4
>> 28 & 0x1;
646 victim
= (tlbsel
== 0) ? gtlb0_get_next_victim(vcpu_e500
) : 0;
648 vcpu
->arch
.shared
->mas0
= MAS0_TLBSEL(tlbsel
)
650 | MAS0_NV(vcpu_e500
->gtlb_nv
[tlbsel
]);
651 vcpu
->arch
.shared
->mas1
=
652 (vcpu
->arch
.shared
->mas6
& MAS6_SPID0
)
653 | (vcpu
->arch
.shared
->mas6
& (MAS6_SAS
? MAS1_TS
: 0))
654 | (vcpu
->arch
.shared
->mas4
& MAS4_TSIZED(~0));
655 vcpu
->arch
.shared
->mas2
&= MAS2_EPN
;
656 vcpu
->arch
.shared
->mas2
|= vcpu
->arch
.shared
->mas4
&
658 vcpu
->arch
.shared
->mas7_3
&= MAS3_U0
| MAS3_U1
|
662 kvmppc_set_exit_type(vcpu
, EMULATED_TLBSX_EXITS
);
666 /* sesel is for tlb1 only */
667 static void write_stlbe(struct kvmppc_vcpu_e500
*vcpu_e500
,
668 struct kvm_book3e_206_tlb_entry
*gtlbe
,
669 struct kvm_book3e_206_tlb_entry
*stlbe
,
670 int stlbsel
, int sesel
)
675 stid
= kvmppc_e500_get_tlb_stid(&vcpu_e500
->vcpu
, gtlbe
);
677 stlbe
->mas1
|= MAS1_TID(stid
);
678 write_host_tlbe(vcpu_e500
, stlbsel
, sesel
, stlbe
);
682 int kvmppc_e500_emul_tlbwe(struct kvm_vcpu
*vcpu
)
684 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
685 struct kvm_book3e_206_tlb_entry
*gtlbe
, stlbe
;
686 int tlbsel
, esel
, stlbsel
, sesel
;
688 tlbsel
= get_tlb_tlbsel(vcpu
);
689 esel
= get_tlb_esel(vcpu
, tlbsel
);
691 gtlbe
= get_entry(vcpu_e500
, tlbsel
, esel
);
693 if (get_tlb_v(gtlbe
))
694 inval_gtlbe_on_host(vcpu_e500
, tlbsel
, esel
);
696 gtlbe
->mas1
= vcpu
->arch
.shared
->mas1
;
697 gtlbe
->mas2
= vcpu
->arch
.shared
->mas2
;
698 gtlbe
->mas7_3
= vcpu
->arch
.shared
->mas7_3
;
700 trace_kvm_booke206_gtlb_write(vcpu
->arch
.shared
->mas0
, gtlbe
->mas1
,
701 gtlbe
->mas2
, gtlbe
->mas7_3
);
703 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */
704 if (tlbe_is_host_safe(vcpu
, gtlbe
)) {
711 gtlbe
->mas1
&= ~MAS1_TSIZE(~0);
712 gtlbe
->mas1
|= MAS1_TSIZE(BOOK3E_PAGESZ_4K
);
715 kvmppc_e500_tlb0_map(vcpu_e500
, esel
, &stlbe
);
716 sesel
= 0; /* unused */
722 eaddr
= get_tlb_eaddr(gtlbe
);
723 raddr
= get_tlb_raddr(gtlbe
);
725 /* Create a 4KB mapping on the host.
726 * If the guest wanted a large page,
727 * only the first 4KB is mapped here and the rest
728 * are mapped on the fly. */
730 sesel
= kvmppc_e500_tlb1_map(vcpu_e500
, eaddr
,
731 raddr
>> PAGE_SHIFT
, gtlbe
, &stlbe
);
738 write_stlbe(vcpu_e500
, gtlbe
, &stlbe
, stlbsel
, sesel
);
741 kvmppc_set_exit_type(vcpu
, EMULATED_TLBWE_EXITS
);
745 static int kvmppc_e500_tlb_search(struct kvm_vcpu
*vcpu
,
746 gva_t eaddr
, unsigned int pid
, int as
)
748 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
751 for (tlbsel
= 0; tlbsel
< 2; tlbsel
++) {
752 esel
= kvmppc_e500_tlb_index(vcpu_e500
, eaddr
, tlbsel
, pid
, as
);
754 return index_of(tlbsel
, esel
);
760 /* 'linear_address' is actually an encoding of AS|PID|EADDR . */
761 int kvmppc_core_vcpu_translate(struct kvm_vcpu
*vcpu
,
762 struct kvm_translation
*tr
)
769 eaddr
= tr
->linear_address
;
770 pid
= (tr
->linear_address
>> 32) & 0xff;
771 as
= (tr
->linear_address
>> 40) & 0x1;
773 index
= kvmppc_e500_tlb_search(vcpu
, eaddr
, pid
, as
);
779 tr
->physical_address
= kvmppc_mmu_xlate(vcpu
, index
, eaddr
);
780 /* XXX what does "writeable" and "usermode" even mean? */
787 int kvmppc_mmu_itlb_index(struct kvm_vcpu
*vcpu
, gva_t eaddr
)
789 unsigned int as
= !!(vcpu
->arch
.shared
->msr
& MSR_IS
);
791 return kvmppc_e500_tlb_search(vcpu
, eaddr
, get_cur_pid(vcpu
), as
);
794 int kvmppc_mmu_dtlb_index(struct kvm_vcpu
*vcpu
, gva_t eaddr
)
796 unsigned int as
= !!(vcpu
->arch
.shared
->msr
& MSR_DS
);
798 return kvmppc_e500_tlb_search(vcpu
, eaddr
, get_cur_pid(vcpu
), as
);
801 void kvmppc_mmu_itlb_miss(struct kvm_vcpu
*vcpu
)
803 unsigned int as
= !!(vcpu
->arch
.shared
->msr
& MSR_IS
);
805 kvmppc_e500_deliver_tlb_miss(vcpu
, vcpu
->arch
.pc
, as
);
808 void kvmppc_mmu_dtlb_miss(struct kvm_vcpu
*vcpu
)
810 unsigned int as
= !!(vcpu
->arch
.shared
->msr
& MSR_DS
);
812 kvmppc_e500_deliver_tlb_miss(vcpu
, vcpu
->arch
.fault_dear
, as
);
815 gpa_t
kvmppc_mmu_xlate(struct kvm_vcpu
*vcpu
, unsigned int index
,
818 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
819 struct kvm_book3e_206_tlb_entry
*gtlbe
;
822 gtlbe
= get_entry(vcpu_e500
, tlbsel_of(index
), esel_of(index
));
823 pgmask
= get_tlb_bytes(gtlbe
) - 1;
825 return get_tlb_raddr(gtlbe
) | (eaddr
& pgmask
);
828 void kvmppc_mmu_destroy(struct kvm_vcpu
*vcpu
)
832 void kvmppc_mmu_map(struct kvm_vcpu
*vcpu
, u64 eaddr
, gpa_t gpaddr
,
835 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
836 struct tlbe_priv
*priv
;
837 struct kvm_book3e_206_tlb_entry
*gtlbe
, stlbe
;
838 int tlbsel
= tlbsel_of(index
);
839 int esel
= esel_of(index
);
842 gtlbe
= get_entry(vcpu_e500
, tlbsel
, esel
);
847 sesel
= 0; /* unused */
848 priv
= &vcpu_e500
->gtlb_priv
[tlbsel
][esel
];
850 kvmppc_e500_setup_stlbe(vcpu
, gtlbe
, BOOK3E_PAGESZ_4K
,
851 &priv
->ref
, eaddr
, &stlbe
);
855 gfn_t gfn
= gpaddr
>> PAGE_SHIFT
;
858 sesel
= kvmppc_e500_tlb1_map(vcpu_e500
, eaddr
, gfn
,
868 write_stlbe(vcpu_e500
, gtlbe
, &stlbe
, stlbsel
, sesel
);
871 static void free_gtlb(struct kvmppc_vcpu_e500
*vcpu_e500
)
875 clear_tlb_refs(vcpu_e500
);
876 kfree(vcpu_e500
->gtlb_priv
[0]);
877 kfree(vcpu_e500
->gtlb_priv
[1]);
879 if (vcpu_e500
->shared_tlb_pages
) {
880 vfree((void *)(round_down((uintptr_t)vcpu_e500
->gtlb_arch
,
883 for (i
= 0; i
< vcpu_e500
->num_shared_tlb_pages
; i
++) {
884 set_page_dirty_lock(vcpu_e500
->shared_tlb_pages
[i
]);
885 put_page(vcpu_e500
->shared_tlb_pages
[i
]);
888 vcpu_e500
->num_shared_tlb_pages
= 0;
889 vcpu_e500
->shared_tlb_pages
= NULL
;
891 kfree(vcpu_e500
->gtlb_arch
);
894 vcpu_e500
->gtlb_arch
= NULL
;
897 void kvmppc_get_sregs_e500_tlb(struct kvm_vcpu
*vcpu
, struct kvm_sregs
*sregs
)
899 sregs
->u
.e
.mas0
= vcpu
->arch
.shared
->mas0
;
900 sregs
->u
.e
.mas1
= vcpu
->arch
.shared
->mas1
;
901 sregs
->u
.e
.mas2
= vcpu
->arch
.shared
->mas2
;
902 sregs
->u
.e
.mas7_3
= vcpu
->arch
.shared
->mas7_3
;
903 sregs
->u
.e
.mas4
= vcpu
->arch
.shared
->mas4
;
904 sregs
->u
.e
.mas6
= vcpu
->arch
.shared
->mas6
;
906 sregs
->u
.e
.mmucfg
= vcpu
->arch
.mmucfg
;
907 sregs
->u
.e
.tlbcfg
[0] = vcpu
->arch
.tlbcfg
[0];
908 sregs
->u
.e
.tlbcfg
[1] = vcpu
->arch
.tlbcfg
[1];
909 sregs
->u
.e
.tlbcfg
[2] = 0;
910 sregs
->u
.e
.tlbcfg
[3] = 0;
913 int kvmppc_set_sregs_e500_tlb(struct kvm_vcpu
*vcpu
, struct kvm_sregs
*sregs
)
915 if (sregs
->u
.e
.features
& KVM_SREGS_E_ARCH206_MMU
) {
916 vcpu
->arch
.shared
->mas0
= sregs
->u
.e
.mas0
;
917 vcpu
->arch
.shared
->mas1
= sregs
->u
.e
.mas1
;
918 vcpu
->arch
.shared
->mas2
= sregs
->u
.e
.mas2
;
919 vcpu
->arch
.shared
->mas7_3
= sregs
->u
.e
.mas7_3
;
920 vcpu
->arch
.shared
->mas4
= sregs
->u
.e
.mas4
;
921 vcpu
->arch
.shared
->mas6
= sregs
->u
.e
.mas6
;
927 int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu
*vcpu
,
928 struct kvm_config_tlb
*cfg
)
930 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
931 struct kvm_book3e_206_tlb_params params
;
934 struct tlbe_priv
*privs
[2] = {};
937 int num_pages
, ret
, i
;
939 if (cfg
->mmu_type
!= KVM_MMU_FSL_BOOKE_NOHV
)
942 if (copy_from_user(¶ms
, (void __user
*)(uintptr_t)cfg
->params
,
946 if (params
.tlb_sizes
[1] > 64)
948 if (params
.tlb_ways
[1] != params
.tlb_sizes
[1])
950 if (params
.tlb_sizes
[2] != 0 || params
.tlb_sizes
[3] != 0)
952 if (params
.tlb_ways
[2] != 0 || params
.tlb_ways
[3] != 0)
955 if (!is_power_of_2(params
.tlb_ways
[0]))
958 sets
= params
.tlb_sizes
[0] >> ilog2(params
.tlb_ways
[0]);
959 if (!is_power_of_2(sets
))
962 array_len
= params
.tlb_sizes
[0] + params
.tlb_sizes
[1];
963 array_len
*= sizeof(struct kvm_book3e_206_tlb_entry
);
965 if (cfg
->array_len
< array_len
)
968 num_pages
= DIV_ROUND_UP(cfg
->array
+ array_len
- 1, PAGE_SIZE
) -
969 cfg
->array
/ PAGE_SIZE
;
970 pages
= kmalloc(sizeof(struct page
*) * num_pages
, GFP_KERNEL
);
974 ret
= get_user_pages_fast(cfg
->array
, num_pages
, 1, pages
);
978 if (ret
!= num_pages
) {
984 virt
= vmap(pages
, num_pages
, VM_MAP
, PAGE_KERNEL
);
988 privs
[0] = kzalloc(sizeof(struct tlbe_priv
) * params
.tlb_sizes
[0],
990 privs
[1] = kzalloc(sizeof(struct tlbe_priv
) * params
.tlb_sizes
[1],
993 if (!privs
[0] || !privs
[1])
996 free_gtlb(vcpu_e500
);
998 vcpu_e500
->gtlb_priv
[0] = privs
[0];
999 vcpu_e500
->gtlb_priv
[1] = privs
[1];
1001 vcpu_e500
->gtlb_arch
= (struct kvm_book3e_206_tlb_entry
*)
1002 (virt
+ (cfg
->array
& (PAGE_SIZE
- 1)));
1004 vcpu_e500
->gtlb_params
[0].entries
= params
.tlb_sizes
[0];
1005 vcpu_e500
->gtlb_params
[1].entries
= params
.tlb_sizes
[1];
1007 vcpu_e500
->gtlb_offset
[0] = 0;
1008 vcpu_e500
->gtlb_offset
[1] = params
.tlb_sizes
[0];
1010 vcpu
->arch
.mmucfg
= mfspr(SPRN_MMUCFG
) & ~MMUCFG_LPIDSIZE
;
1012 vcpu
->arch
.tlbcfg
[0] &= ~(TLBnCFG_N_ENTRY
| TLBnCFG_ASSOC
);
1013 if (params
.tlb_sizes
[0] <= 2048)
1014 vcpu
->arch
.tlbcfg
[0] |= params
.tlb_sizes
[0];
1015 vcpu
->arch
.tlbcfg
[0] |= params
.tlb_ways
[0] << TLBnCFG_ASSOC_SHIFT
;
1017 vcpu
->arch
.tlbcfg
[1] &= ~(TLBnCFG_N_ENTRY
| TLBnCFG_ASSOC
);
1018 vcpu
->arch
.tlbcfg
[1] |= params
.tlb_sizes
[1];
1019 vcpu
->arch
.tlbcfg
[1] |= params
.tlb_ways
[1] << TLBnCFG_ASSOC_SHIFT
;
1021 vcpu_e500
->shared_tlb_pages
= pages
;
1022 vcpu_e500
->num_shared_tlb_pages
= num_pages
;
1024 vcpu_e500
->gtlb_params
[0].ways
= params
.tlb_ways
[0];
1025 vcpu_e500
->gtlb_params
[0].sets
= sets
;
1027 vcpu_e500
->gtlb_params
[1].ways
= params
.tlb_sizes
[1];
1028 vcpu_e500
->gtlb_params
[1].sets
= 1;
1036 for (i
= 0; i
< num_pages
; i
++)
1044 int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu
*vcpu
,
1045 struct kvm_dirty_tlb
*dirty
)
1047 struct kvmppc_vcpu_e500
*vcpu_e500
= to_e500(vcpu
);
1049 clear_tlb_refs(vcpu_e500
);
1053 int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500
*vcpu_e500
)
1055 struct kvm_vcpu
*vcpu
= &vcpu_e500
->vcpu
;
1056 int entry_size
= sizeof(struct kvm_book3e_206_tlb_entry
);
1057 int entries
= KVM_E500_TLB0_SIZE
+ KVM_E500_TLB1_SIZE
;
1059 host_tlb_params
[0].entries
= mfspr(SPRN_TLB0CFG
) & TLBnCFG_N_ENTRY
;
1060 host_tlb_params
[1].entries
= mfspr(SPRN_TLB1CFG
) & TLBnCFG_N_ENTRY
;
1063 * This should never happen on real e500 hardware, but is
1064 * architecturally possible -- e.g. in some weird nested
1065 * virtualization case.
1067 if (host_tlb_params
[0].entries
== 0 ||
1068 host_tlb_params
[1].entries
== 0) {
1069 pr_err("%s: need to know host tlb size\n", __func__
);
1073 host_tlb_params
[0].ways
= (mfspr(SPRN_TLB0CFG
) & TLBnCFG_ASSOC
) >>
1074 TLBnCFG_ASSOC_SHIFT
;
1075 host_tlb_params
[1].ways
= host_tlb_params
[1].entries
;
1077 if (!is_power_of_2(host_tlb_params
[0].entries
) ||
1078 !is_power_of_2(host_tlb_params
[0].ways
) ||
1079 host_tlb_params
[0].entries
< host_tlb_params
[0].ways
||
1080 host_tlb_params
[0].ways
== 0) {
1081 pr_err("%s: bad tlb0 host config: %u entries %u ways\n",
1082 __func__
, host_tlb_params
[0].entries
,
1083 host_tlb_params
[0].ways
);
1087 host_tlb_params
[0].sets
=
1088 host_tlb_params
[0].entries
/ host_tlb_params
[0].ways
;
1089 host_tlb_params
[1].sets
= 1;
1091 vcpu_e500
->gtlb_params
[0].entries
= KVM_E500_TLB0_SIZE
;
1092 vcpu_e500
->gtlb_params
[1].entries
= KVM_E500_TLB1_SIZE
;
1094 vcpu_e500
->gtlb_params
[0].ways
= KVM_E500_TLB0_WAY_NUM
;
1095 vcpu_e500
->gtlb_params
[0].sets
=
1096 KVM_E500_TLB0_SIZE
/ KVM_E500_TLB0_WAY_NUM
;
1098 vcpu_e500
->gtlb_params
[1].ways
= KVM_E500_TLB1_SIZE
;
1099 vcpu_e500
->gtlb_params
[1].sets
= 1;
1101 vcpu_e500
->gtlb_arch
= kmalloc(entries
* entry_size
, GFP_KERNEL
);
1102 if (!vcpu_e500
->gtlb_arch
)
1105 vcpu_e500
->gtlb_offset
[0] = 0;
1106 vcpu_e500
->gtlb_offset
[1] = KVM_E500_TLB0_SIZE
;
1108 vcpu_e500
->tlb_refs
[0] =
1109 kzalloc(sizeof(struct tlbe_ref
) * host_tlb_params
[0].entries
,
1111 if (!vcpu_e500
->tlb_refs
[0])
1114 vcpu_e500
->tlb_refs
[1] =
1115 kzalloc(sizeof(struct tlbe_ref
) * host_tlb_params
[1].entries
,
1117 if (!vcpu_e500
->tlb_refs
[1])
1120 vcpu_e500
->gtlb_priv
[0] = kzalloc(sizeof(struct tlbe_ref
) *
1121 vcpu_e500
->gtlb_params
[0].entries
,
1123 if (!vcpu_e500
->gtlb_priv
[0])
1126 vcpu_e500
->gtlb_priv
[1] = kzalloc(sizeof(struct tlbe_ref
) *
1127 vcpu_e500
->gtlb_params
[1].entries
,
1129 if (!vcpu_e500
->gtlb_priv
[1])
1132 /* Init TLB configuration register */
1133 vcpu
->arch
.tlbcfg
[0] = mfspr(SPRN_TLB0CFG
) &
1134 ~(TLBnCFG_N_ENTRY
| TLBnCFG_ASSOC
);
1135 vcpu
->arch
.tlbcfg
[0] |= vcpu_e500
->gtlb_params
[0].entries
;
1136 vcpu
->arch
.tlbcfg
[0] |=
1137 vcpu_e500
->gtlb_params
[0].ways
<< TLBnCFG_ASSOC_SHIFT
;
1139 vcpu
->arch
.tlbcfg
[1] = mfspr(SPRN_TLB1CFG
) &
1140 ~(TLBnCFG_N_ENTRY
| TLBnCFG_ASSOC
);
1141 vcpu
->arch
.tlbcfg
[0] |= vcpu_e500
->gtlb_params
[1].entries
;
1142 vcpu
->arch
.tlbcfg
[0] |=
1143 vcpu_e500
->gtlb_params
[1].ways
<< TLBnCFG_ASSOC_SHIFT
;
1148 free_gtlb(vcpu_e500
);
1149 kfree(vcpu_e500
->tlb_refs
[0]);
1150 kfree(vcpu_e500
->tlb_refs
[1]);
1154 void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500
*vcpu_e500
)
1156 free_gtlb(vcpu_e500
);
1157 kfree(vcpu_e500
->tlb_refs
[0]);
1158 kfree(vcpu_e500
->tlb_refs
[1]);