2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
18 #include <linux/types.h>
19 #include <linux/string.h>
20 #include <linux/kvm.h>
21 #include <linux/kvm_host.h>
22 #include <linux/highmem.h>
23 #include <linux/gfp.h>
24 #include <linux/slab.h>
25 #include <linux/hugetlb.h>
26 #include <linux/vmalloc.h>
27 #include <linux/srcu.h>
29 #include <asm/tlbflush.h>
30 #include <asm/kvm_ppc.h>
31 #include <asm/kvm_book3s.h>
32 #include <asm/mmu-hash64.h>
33 #include <asm/hvcall.h>
34 #include <asm/synch.h>
35 #include <asm/ppc-opcode.h>
36 #include <asm/cputable.h>
38 /* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */
39 #define MAX_LPID_970 63
41 /* Power architecture requires HPT is at least 256kB */
42 #define PPC_MIN_HPT_ORDER 18
44 static long kvmppc_virtmode_do_h_enter(struct kvm
*kvm
, unsigned long flags
,
45 long pte_index
, unsigned long pteh
,
46 unsigned long ptel
, unsigned long *pte_idx_ret
);
48 long kvmppc_alloc_hpt(struct kvm
*kvm
, u32
*htab_orderp
)
51 struct revmap_entry
*rev
;
52 struct kvmppc_linear_info
*li
;
53 long order
= kvm_hpt_order
;
57 if (order
< PPC_MIN_HPT_ORDER
)
58 order
= PPC_MIN_HPT_ORDER
;
62 * If the user wants a different size from default,
63 * try first to allocate it from the kernel page allocator.
66 if (order
!= kvm_hpt_order
) {
67 hpt
= __get_free_pages(GFP_KERNEL
|__GFP_ZERO
|__GFP_REPEAT
|
68 __GFP_NOWARN
, order
- PAGE_SHIFT
);
73 /* Next try to allocate from the preallocated pool */
77 hpt
= (ulong
)li
->base_virt
;
78 kvm
->arch
.hpt_li
= li
;
79 order
= kvm_hpt_order
;
83 /* Lastly try successively smaller sizes from the page allocator */
84 while (!hpt
&& order
> PPC_MIN_HPT_ORDER
) {
85 hpt
= __get_free_pages(GFP_KERNEL
|__GFP_ZERO
|__GFP_REPEAT
|
86 __GFP_NOWARN
, order
- PAGE_SHIFT
);
94 kvm
->arch
.hpt_virt
= hpt
;
95 kvm
->arch
.hpt_order
= order
;
96 /* HPTEs are 2**4 bytes long */
97 kvm
->arch
.hpt_npte
= 1ul << (order
- 4);
98 /* 128 (2**7) bytes in each HPTEG */
99 kvm
->arch
.hpt_mask
= (1ul << (order
- 7)) - 1;
101 /* Allocate reverse map array */
102 rev
= vmalloc(sizeof(struct revmap_entry
) * kvm
->arch
.hpt_npte
);
104 pr_err("kvmppc_alloc_hpt: Couldn't alloc reverse map array\n");
107 kvm
->arch
.revmap
= rev
;
108 kvm
->arch
.sdr1
= __pa(hpt
) | (order
- 18);
110 pr_info("KVM guest htab at %lx (order %ld), LPID %x\n",
111 hpt
, order
, kvm
->arch
.lpid
);
114 *htab_orderp
= order
;
118 if (kvm
->arch
.hpt_li
)
119 kvm_release_hpt(kvm
->arch
.hpt_li
);
121 free_pages(hpt
, order
- PAGE_SHIFT
);
125 long kvmppc_alloc_reset_hpt(struct kvm
*kvm
, u32
*htab_orderp
)
130 mutex_lock(&kvm
->lock
);
131 if (kvm
->arch
.rma_setup_done
) {
132 kvm
->arch
.rma_setup_done
= 0;
133 /* order rma_setup_done vs. vcpus_running */
135 if (atomic_read(&kvm
->arch
.vcpus_running
)) {
136 kvm
->arch
.rma_setup_done
= 1;
140 if (kvm
->arch
.hpt_virt
) {
141 order
= kvm
->arch
.hpt_order
;
142 /* Set the entire HPT to 0, i.e. invalid HPTEs */
143 memset((void *)kvm
->arch
.hpt_virt
, 0, 1ul << order
);
145 * Set the whole last_vcpu array to an invalid vcpu number.
146 * This ensures that each vcpu will flush its TLB on next entry.
148 memset(kvm
->arch
.last_vcpu
, 0xff, sizeof(kvm
->arch
.last_vcpu
));
149 *htab_orderp
= order
;
152 err
= kvmppc_alloc_hpt(kvm
, htab_orderp
);
153 order
= *htab_orderp
;
156 mutex_unlock(&kvm
->lock
);
160 void kvmppc_free_hpt(struct kvm
*kvm
)
162 kvmppc_free_lpid(kvm
->arch
.lpid
);
163 vfree(kvm
->arch
.revmap
);
164 if (kvm
->arch
.hpt_li
)
165 kvm_release_hpt(kvm
->arch
.hpt_li
);
167 free_pages(kvm
->arch
.hpt_virt
,
168 kvm
->arch
.hpt_order
- PAGE_SHIFT
);
171 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
172 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize
)
174 return (pgsize
> 0x1000) ? HPTE_V_LARGE
: 0;
177 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
178 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize
)
180 return (pgsize
== 0x10000) ? 0x1000 : 0;
183 void kvmppc_map_vrma(struct kvm_vcpu
*vcpu
, struct kvm_memory_slot
*memslot
,
184 unsigned long porder
)
187 unsigned long npages
;
188 unsigned long hp_v
, hp_r
;
189 unsigned long addr
, hash
;
191 unsigned long hp0
, hp1
;
192 unsigned long idx_ret
;
194 struct kvm
*kvm
= vcpu
->kvm
;
196 psize
= 1ul << porder
;
197 npages
= memslot
->npages
>> (porder
- PAGE_SHIFT
);
199 /* VRMA can't be > 1TB */
200 if (npages
> 1ul << (40 - porder
))
201 npages
= 1ul << (40 - porder
);
202 /* Can't use more than 1 HPTE per HPTEG */
203 if (npages
> kvm
->arch
.hpt_mask
+ 1)
204 npages
= kvm
->arch
.hpt_mask
+ 1;
206 hp0
= HPTE_V_1TB_SEG
| (VRMA_VSID
<< (40 - 16)) |
207 HPTE_V_BOLTED
| hpte0_pgsize_encoding(psize
);
208 hp1
= hpte1_pgsize_encoding(psize
) |
209 HPTE_R_R
| HPTE_R_C
| HPTE_R_M
| PP_RWXX
;
211 for (i
= 0; i
< npages
; ++i
) {
213 /* can't use hpt_hash since va > 64 bits */
214 hash
= (i
^ (VRMA_VSID
^ (VRMA_VSID
<< 25))) & kvm
->arch
.hpt_mask
;
216 * We assume that the hash table is empty and no
217 * vcpus are using it at this stage. Since we create
218 * at most one HPTE per HPTEG, we just assume entry 7
219 * is available and use it.
221 hash
= (hash
<< 3) + 7;
222 hp_v
= hp0
| ((addr
>> 16) & ~0x7fUL
);
224 ret
= kvmppc_virtmode_do_h_enter(kvm
, H_EXACT
, hash
, hp_v
, hp_r
,
226 if (ret
!= H_SUCCESS
) {
227 pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
234 int kvmppc_mmu_hv_init(void)
236 unsigned long host_lpid
, rsvd_lpid
;
238 if (!cpu_has_feature(CPU_FTR_HVMODE
))
241 /* POWER7 has 10-bit LPIDs, PPC970 and e500mc have 6-bit LPIDs */
242 if (cpu_has_feature(CPU_FTR_ARCH_206
)) {
243 host_lpid
= mfspr(SPRN_LPID
); /* POWER7 */
244 rsvd_lpid
= LPID_RSVD
;
246 host_lpid
= 0; /* PPC970 */
247 rsvd_lpid
= MAX_LPID_970
;
250 kvmppc_init_lpid(rsvd_lpid
+ 1);
252 kvmppc_claim_lpid(host_lpid
);
253 /* rsvd_lpid is reserved for use in partition switching */
254 kvmppc_claim_lpid(rsvd_lpid
);
259 void kvmppc_mmu_destroy(struct kvm_vcpu
*vcpu
)
263 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu
*vcpu
)
265 kvmppc_set_msr(vcpu
, MSR_SF
| MSR_ME
);
269 * This is called to get a reference to a guest page if there isn't
270 * one already in the memslot->arch.slot_phys[] array.
272 static long kvmppc_get_guest_page(struct kvm
*kvm
, unsigned long gfn
,
273 struct kvm_memory_slot
*memslot
,
278 struct page
*page
, *hpage
, *pages
[1];
279 unsigned long s
, pgsize
;
280 unsigned long *physp
;
281 unsigned int is_io
, got
, pgorder
;
282 struct vm_area_struct
*vma
;
283 unsigned long pfn
, i
, npages
;
285 physp
= memslot
->arch
.slot_phys
;
288 if (physp
[gfn
- memslot
->base_gfn
])
296 start
= gfn_to_hva_memslot(memslot
, gfn
);
298 /* Instantiate and get the page we want access to */
299 np
= get_user_pages_fast(start
, 1, 1, pages
);
301 /* Look up the vma for the page */
302 down_read(¤t
->mm
->mmap_sem
);
303 vma
= find_vma(current
->mm
, start
);
304 if (!vma
|| vma
->vm_start
> start
||
305 start
+ psize
> vma
->vm_end
||
306 !(vma
->vm_flags
& VM_PFNMAP
))
308 is_io
= hpte_cache_bits(pgprot_val(vma
->vm_page_prot
));
309 pfn
= vma
->vm_pgoff
+ ((start
- vma
->vm_start
) >> PAGE_SHIFT
);
310 /* check alignment of pfn vs. requested page size */
311 if (psize
> PAGE_SIZE
&& (pfn
& ((psize
>> PAGE_SHIFT
) - 1)))
313 up_read(¤t
->mm
->mmap_sem
);
317 got
= KVMPPC_GOT_PAGE
;
319 /* See if this is a large page */
321 if (PageHuge(page
)) {
322 hpage
= compound_head(page
);
323 s
<<= compound_order(hpage
);
324 /* Get the whole large page if slot alignment is ok */
325 if (s
> psize
&& slot_is_aligned(memslot
, s
) &&
326 !(memslot
->userspace_addr
& (s
- 1))) {
336 pfn
= page_to_pfn(page
);
339 npages
= pgsize
>> PAGE_SHIFT
;
340 pgorder
= __ilog2(npages
);
341 physp
+= (gfn
- memslot
->base_gfn
) & ~(npages
- 1);
342 spin_lock(&kvm
->arch
.slot_phys_lock
);
343 for (i
= 0; i
< npages
; ++i
) {
345 physp
[i
] = ((pfn
+ i
) << PAGE_SHIFT
) +
346 got
+ is_io
+ pgorder
;
350 spin_unlock(&kvm
->arch
.slot_phys_lock
);
359 up_read(¤t
->mm
->mmap_sem
);
363 long kvmppc_virtmode_do_h_enter(struct kvm
*kvm
, unsigned long flags
,
364 long pte_index
, unsigned long pteh
,
365 unsigned long ptel
, unsigned long *pte_idx_ret
)
367 unsigned long psize
, gpa
, gfn
;
368 struct kvm_memory_slot
*memslot
;
371 if (kvm
->arch
.using_mmu_notifiers
)
374 psize
= hpte_page_size(pteh
, ptel
);
378 pteh
&= ~(HPTE_V_HVLOCK
| HPTE_V_ABSENT
| HPTE_V_VALID
);
380 /* Find the memslot (if any) for this address */
381 gpa
= (ptel
& HPTE_R_RPN
) & ~(psize
- 1);
382 gfn
= gpa
>> PAGE_SHIFT
;
383 memslot
= gfn_to_memslot(kvm
, gfn
);
384 if (memslot
&& !(memslot
->flags
& KVM_MEMSLOT_INVALID
)) {
385 if (!slot_is_aligned(memslot
, psize
))
387 if (kvmppc_get_guest_page(kvm
, gfn
, memslot
, psize
) < 0)
392 /* Protect linux PTE lookup from page table destruction */
393 rcu_read_lock_sched(); /* this disables preemption too */
394 ret
= kvmppc_do_h_enter(kvm
, flags
, pte_index
, pteh
, ptel
,
395 current
->mm
->pgd
, false, pte_idx_ret
);
396 rcu_read_unlock_sched();
397 if (ret
== H_TOO_HARD
) {
398 /* this can't happen */
399 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
400 ret
= H_RESOURCE
; /* or something */
407 * We come here on a H_ENTER call from the guest when we are not
408 * using mmu notifiers and we don't have the requested page pinned
411 long kvmppc_virtmode_h_enter(struct kvm_vcpu
*vcpu
, unsigned long flags
,
412 long pte_index
, unsigned long pteh
,
415 return kvmppc_virtmode_do_h_enter(vcpu
->kvm
, flags
, pte_index
,
416 pteh
, ptel
, &vcpu
->arch
.gpr
[4]);
419 static struct kvmppc_slb
*kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu
*vcpu
,
425 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
426 if (!(vcpu
->arch
.slb
[i
].orige
& SLB_ESID_V
))
429 if (vcpu
->arch
.slb
[i
].origv
& SLB_VSID_B_1T
)
434 if (((vcpu
->arch
.slb
[i
].orige
^ eaddr
) & mask
) == 0)
435 return &vcpu
->arch
.slb
[i
];
440 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v
, unsigned long r
,
443 unsigned long ra_mask
;
445 ra_mask
= hpte_page_size(v
, r
) - 1;
446 return (r
& HPTE_R_RPN
& ~ra_mask
) | (ea
& ra_mask
);
449 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
450 struct kvmppc_pte
*gpte
, bool data
)
452 struct kvm
*kvm
= vcpu
->kvm
;
453 struct kvmppc_slb
*slbe
;
455 unsigned long pp
, key
;
457 unsigned long *hptep
;
459 int virtmode
= vcpu
->arch
.shregs
.msr
& (data
? MSR_DR
: MSR_IR
);
463 slbe
= kvmppc_mmu_book3s_hv_find_slbe(vcpu
, eaddr
);
468 /* real mode access */
469 slb_v
= vcpu
->kvm
->arch
.vrma_slb_v
;
472 /* Find the HPTE in the hash table */
473 index
= kvmppc_hv_find_lock_hpte(kvm
, eaddr
, slb_v
,
474 HPTE_V_VALID
| HPTE_V_ABSENT
);
477 hptep
= (unsigned long *)(kvm
->arch
.hpt_virt
+ (index
<< 4));
478 v
= hptep
[0] & ~HPTE_V_HVLOCK
;
479 gr
= kvm
->arch
.revmap
[index
].guest_rpte
;
481 /* Unlock the HPTE */
482 asm volatile("lwsync" : : : "memory");
486 gpte
->vpage
= ((v
& HPTE_V_AVPN
) << 4) | ((eaddr
>> 12) & 0xfff);
488 /* Get PP bits and key for permission check */
489 pp
= gr
& (HPTE_R_PP0
| HPTE_R_PP
);
490 key
= (vcpu
->arch
.shregs
.msr
& MSR_PR
) ? SLB_VSID_KP
: SLB_VSID_KS
;
493 /* Calculate permissions */
494 gpte
->may_read
= hpte_read_permission(pp
, key
);
495 gpte
->may_write
= hpte_write_permission(pp
, key
);
496 gpte
->may_execute
= gpte
->may_read
&& !(gr
& (HPTE_R_N
| HPTE_R_G
));
498 /* Storage key permission check for POWER7 */
499 if (data
&& virtmode
&& cpu_has_feature(CPU_FTR_ARCH_206
)) {
500 int amrfield
= hpte_get_skey_perm(gr
, vcpu
->arch
.amr
);
507 /* Get the guest physical address */
508 gpte
->raddr
= kvmppc_mmu_get_real_addr(v
, gr
, eaddr
);
513 * Quick test for whether an instruction is a load or a store.
514 * If the instruction is a load or a store, then this will indicate
515 * which it is, at least on server processors. (Embedded processors
516 * have some external PID instructions that don't follow the rule
517 * embodied here.) If the instruction isn't a load or store, then
518 * this doesn't return anything useful.
520 static int instruction_is_store(unsigned int instr
)
525 if ((instr
& 0xfc000000) == 0x7c000000)
526 mask
= 0x100; /* major opcode 31 */
527 return (instr
& mask
) != 0;
530 static int kvmppc_hv_emulate_mmio(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
531 unsigned long gpa
, gva_t ea
, int is_store
)
535 unsigned long srr0
= kvmppc_get_pc(vcpu
);
537 /* We try to load the last instruction. We don't let
538 * emulate_instruction do it as it doesn't check what
540 * If we fail, we just return to the guest and try executing it again.
542 if (vcpu
->arch
.last_inst
== KVM_INST_FETCH_FAILED
) {
543 ret
= kvmppc_ld(vcpu
, &srr0
, sizeof(u32
), &last_inst
, false);
544 if (ret
!= EMULATE_DONE
|| last_inst
== KVM_INST_FETCH_FAILED
)
546 vcpu
->arch
.last_inst
= last_inst
;
550 * WARNING: We do not know for sure whether the instruction we just
551 * read from memory is the same that caused the fault in the first
552 * place. If the instruction we read is neither an load or a store,
553 * then it can't access memory, so we don't need to worry about
554 * enforcing access permissions. So, assuming it is a load or
555 * store, we just check that its direction (load or store) is
556 * consistent with the original fault, since that's what we
557 * checked the access permissions against. If there is a mismatch
558 * we just return and retry the instruction.
561 if (instruction_is_store(vcpu
->arch
.last_inst
) != !!is_store
)
565 * Emulated accesses are emulated by looking at the hash for
566 * translation once, then performing the access later. The
567 * translation could be invalidated in the meantime in which
568 * point performing the subsequent memory access on the old
569 * physical address could possibly be a security hole for the
570 * guest (but not the host).
572 * This is less of an issue for MMIO stores since they aren't
573 * globally visible. It could be an issue for MMIO loads to
574 * a certain extent but we'll ignore it for now.
577 vcpu
->arch
.paddr_accessed
= gpa
;
578 vcpu
->arch
.vaddr_accessed
= ea
;
579 return kvmppc_emulate_mmio(run
, vcpu
);
582 int kvmppc_book3s_hv_page_fault(struct kvm_run
*run
, struct kvm_vcpu
*vcpu
,
583 unsigned long ea
, unsigned long dsisr
)
585 struct kvm
*kvm
= vcpu
->kvm
;
586 unsigned long *hptep
, hpte
[3], r
;
587 unsigned long mmu_seq
, psize
, pte_size
;
588 unsigned long gpa
, gfn
, hva
, pfn
;
589 struct kvm_memory_slot
*memslot
;
591 struct revmap_entry
*rev
;
592 struct page
*page
, *pages
[1];
593 long index
, ret
, npages
;
595 unsigned int writing
, write_ok
;
596 struct vm_area_struct
*vma
;
597 unsigned long rcbits
;
600 * Real-mode code has already searched the HPT and found the
601 * entry we're interested in. Lock the entry and check that
602 * it hasn't changed. If it has, just return and re-execute the
605 if (ea
!= vcpu
->arch
.pgfault_addr
)
607 index
= vcpu
->arch
.pgfault_index
;
608 hptep
= (unsigned long *)(kvm
->arch
.hpt_virt
+ (index
<< 4));
609 rev
= &kvm
->arch
.revmap
[index
];
611 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
613 hpte
[0] = hptep
[0] & ~HPTE_V_HVLOCK
;
615 hpte
[2] = r
= rev
->guest_rpte
;
616 asm volatile("lwsync" : : : "memory");
620 if (hpte
[0] != vcpu
->arch
.pgfault_hpte
[0] ||
621 hpte
[1] != vcpu
->arch
.pgfault_hpte
[1])
624 /* Translate the logical address and get the page */
625 psize
= hpte_page_size(hpte
[0], r
);
626 gpa
= (r
& HPTE_R_RPN
& ~(psize
- 1)) | (ea
& (psize
- 1));
627 gfn
= gpa
>> PAGE_SHIFT
;
628 memslot
= gfn_to_memslot(kvm
, gfn
);
630 /* No memslot means it's an emulated MMIO region */
631 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
632 return kvmppc_hv_emulate_mmio(run
, vcpu
, gpa
, ea
,
633 dsisr
& DSISR_ISSTORE
);
635 if (!kvm
->arch
.using_mmu_notifiers
)
636 return -EFAULT
; /* should never get here */
638 /* used to check for invalidations in progress */
639 mmu_seq
= kvm
->mmu_notifier_seq
;
645 pte_size
= PAGE_SIZE
;
646 writing
= (dsisr
& DSISR_ISSTORE
) != 0;
647 /* If writing != 0, then the HPTE must allow writing, if we get here */
649 hva
= gfn_to_hva_memslot(memslot
, gfn
);
650 npages
= get_user_pages_fast(hva
, 1, writing
, pages
);
652 /* Check if it's an I/O mapping */
653 down_read(¤t
->mm
->mmap_sem
);
654 vma
= find_vma(current
->mm
, hva
);
655 if (vma
&& vma
->vm_start
<= hva
&& hva
+ psize
<= vma
->vm_end
&&
656 (vma
->vm_flags
& VM_PFNMAP
)) {
657 pfn
= vma
->vm_pgoff
+
658 ((hva
- vma
->vm_start
) >> PAGE_SHIFT
);
660 is_io
= hpte_cache_bits(pgprot_val(vma
->vm_page_prot
));
661 write_ok
= vma
->vm_flags
& VM_WRITE
;
663 up_read(¤t
->mm
->mmap_sem
);
668 if (PageHuge(page
)) {
669 page
= compound_head(page
);
670 pte_size
<<= compound_order(page
);
672 /* if the guest wants write access, see if that is OK */
673 if (!writing
&& hpte_is_writable(r
)) {
677 * We need to protect against page table destruction
678 * while looking up and updating the pte.
680 rcu_read_lock_sched();
681 ptep
= find_linux_pte_or_hugepte(current
->mm
->pgd
,
683 if (ptep
&& pte_present(*ptep
)) {
684 pte
= kvmppc_read_update_linux_pte(ptep
, 1);
688 rcu_read_unlock_sched();
690 pfn
= page_to_pfn(page
);
694 if (psize
> pte_size
)
697 /* Check WIMG vs. the actual page we're accessing */
698 if (!hpte_cache_flags_ok(r
, is_io
)) {
702 * Allow guest to map emulated device memory as
703 * uncacheable, but actually make it cacheable.
705 r
= (r
& ~(HPTE_R_W
|HPTE_R_I
|HPTE_R_G
)) | HPTE_R_M
;
708 /* Set the HPTE to point to pfn */
709 r
= (r
& ~(HPTE_R_PP0
- pte_size
)) | (pfn
<< PAGE_SHIFT
);
710 if (hpte_is_writable(r
) && !write_ok
)
711 r
= hpte_make_readonly(r
);
714 while (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
))
716 if ((hptep
[0] & ~HPTE_V_HVLOCK
) != hpte
[0] || hptep
[1] != hpte
[1] ||
717 rev
->guest_rpte
!= hpte
[2])
718 /* HPTE has been changed under us; let the guest retry */
720 hpte
[0] = (hpte
[0] & ~HPTE_V_ABSENT
) | HPTE_V_VALID
;
722 rmap
= &memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
];
725 /* Check if we might have been invalidated; let the guest retry if so */
727 if (mmu_notifier_retry(vcpu
->kvm
, mmu_seq
)) {
732 /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
733 rcbits
= *rmap
>> KVMPPC_RMAP_RC_SHIFT
;
734 r
&= rcbits
| ~(HPTE_R_R
| HPTE_R_C
);
736 if (hptep
[0] & HPTE_V_VALID
) {
737 /* HPTE was previously valid, so we need to invalidate it */
739 hptep
[0] |= HPTE_V_ABSENT
;
740 kvmppc_invalidate_hpte(kvm
, hptep
, index
);
741 /* don't lose previous R and C bits */
742 r
|= hptep
[1] & (HPTE_R_R
| HPTE_R_C
);
744 kvmppc_add_revmap_chain(kvm
, rev
, rmap
, index
, 0);
750 asm volatile("ptesync" : : : "memory");
752 if (page
&& hpte_is_writable(r
))
758 * We drop pages[0] here, not page because page might
759 * have been set to the head page of a compound, but
760 * we have to drop the reference on the correct tail
761 * page to match the get inside gup()
768 hptep
[0] &= ~HPTE_V_HVLOCK
;
773 static int kvm_handle_hva_range(struct kvm
*kvm
,
776 int (*handler
)(struct kvm
*kvm
,
777 unsigned long *rmapp
,
782 struct kvm_memslots
*slots
;
783 struct kvm_memory_slot
*memslot
;
785 slots
= kvm_memslots(kvm
);
786 kvm_for_each_memslot(memslot
, slots
) {
787 unsigned long hva_start
, hva_end
;
790 hva_start
= max(start
, memslot
->userspace_addr
);
791 hva_end
= min(end
, memslot
->userspace_addr
+
792 (memslot
->npages
<< PAGE_SHIFT
));
793 if (hva_start
>= hva_end
)
796 * {gfn(page) | page intersects with [hva_start, hva_end)} =
797 * {gfn, gfn+1, ..., gfn_end-1}.
799 gfn
= hva_to_gfn_memslot(hva_start
, memslot
);
800 gfn_end
= hva_to_gfn_memslot(hva_end
+ PAGE_SIZE
- 1, memslot
);
802 for (; gfn
< gfn_end
; ++gfn
) {
803 gfn_t gfn_offset
= gfn
- memslot
->base_gfn
;
805 ret
= handler(kvm
, &memslot
->arch
.rmap
[gfn_offset
], gfn
);
813 static int kvm_handle_hva(struct kvm
*kvm
, unsigned long hva
,
814 int (*handler
)(struct kvm
*kvm
, unsigned long *rmapp
,
817 return kvm_handle_hva_range(kvm
, hva
, hva
+ 1, handler
);
820 static int kvm_unmap_rmapp(struct kvm
*kvm
, unsigned long *rmapp
,
823 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
824 unsigned long h
, i
, j
;
825 unsigned long *hptep
;
826 unsigned long ptel
, psize
, rcbits
;
830 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
836 * To avoid an ABBA deadlock with the HPTE lock bit,
837 * we can't spin on the HPTE lock while holding the
840 i
= *rmapp
& KVMPPC_RMAP_INDEX
;
841 hptep
= (unsigned long *) (kvm
->arch
.hpt_virt
+ (i
<< 4));
842 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
843 /* unlock rmap before spinning on the HPTE lock */
845 while (hptep
[0] & HPTE_V_HVLOCK
)
851 /* chain is now empty */
852 *rmapp
&= ~(KVMPPC_RMAP_PRESENT
| KVMPPC_RMAP_INDEX
);
854 /* remove i from chain */
858 rev
[i
].forw
= rev
[i
].back
= i
;
859 *rmapp
= (*rmapp
& ~KVMPPC_RMAP_INDEX
) | j
;
862 /* Now check and modify the HPTE */
863 ptel
= rev
[i
].guest_rpte
;
864 psize
= hpte_page_size(hptep
[0], ptel
);
865 if ((hptep
[0] & HPTE_V_VALID
) &&
866 hpte_rpn(ptel
, psize
) == gfn
) {
867 if (kvm
->arch
.using_mmu_notifiers
)
868 hptep
[0] |= HPTE_V_ABSENT
;
869 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
870 /* Harvest R and C */
871 rcbits
= hptep
[1] & (HPTE_R_R
| HPTE_R_C
);
872 *rmapp
|= rcbits
<< KVMPPC_RMAP_RC_SHIFT
;
873 rev
[i
].guest_rpte
= ptel
| rcbits
;
876 hptep
[0] &= ~HPTE_V_HVLOCK
;
881 int kvm_unmap_hva(struct kvm
*kvm
, unsigned long hva
)
883 if (kvm
->arch
.using_mmu_notifiers
)
884 kvm_handle_hva(kvm
, hva
, kvm_unmap_rmapp
);
888 int kvm_unmap_hva_range(struct kvm
*kvm
, unsigned long start
, unsigned long end
)
890 if (kvm
->arch
.using_mmu_notifiers
)
891 kvm_handle_hva_range(kvm
, start
, end
, kvm_unmap_rmapp
);
895 void kvmppc_core_flush_memslot(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
)
897 unsigned long *rmapp
;
901 rmapp
= memslot
->arch
.rmap
;
902 gfn
= memslot
->base_gfn
;
903 for (n
= memslot
->npages
; n
; --n
) {
905 * Testing the present bit without locking is OK because
906 * the memslot has been marked invalid already, and hence
907 * no new HPTEs referencing this page can be created,
908 * thus the present bit can't go from 0 to 1.
910 if (*rmapp
& KVMPPC_RMAP_PRESENT
)
911 kvm_unmap_rmapp(kvm
, rmapp
, gfn
);
917 static int kvm_age_rmapp(struct kvm
*kvm
, unsigned long *rmapp
,
920 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
921 unsigned long head
, i
, j
;
922 unsigned long *hptep
;
927 if (*rmapp
& KVMPPC_RMAP_REFERENCED
) {
928 *rmapp
&= ~KVMPPC_RMAP_REFERENCED
;
931 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
936 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
938 hptep
= (unsigned long *) (kvm
->arch
.hpt_virt
+ (i
<< 4));
941 /* If this HPTE isn't referenced, ignore it */
942 if (!(hptep
[1] & HPTE_R_R
))
945 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
946 /* unlock rmap before spinning on the HPTE lock */
948 while (hptep
[0] & HPTE_V_HVLOCK
)
953 /* Now check and modify the HPTE */
954 if ((hptep
[0] & HPTE_V_VALID
) && (hptep
[1] & HPTE_R_R
)) {
955 kvmppc_clear_ref_hpte(kvm
, hptep
, i
);
956 rev
[i
].guest_rpte
|= HPTE_R_R
;
959 hptep
[0] &= ~HPTE_V_HVLOCK
;
960 } while ((i
= j
) != head
);
966 int kvm_age_hva(struct kvm
*kvm
, unsigned long hva
)
968 if (!kvm
->arch
.using_mmu_notifiers
)
970 return kvm_handle_hva(kvm
, hva
, kvm_age_rmapp
);
973 static int kvm_test_age_rmapp(struct kvm
*kvm
, unsigned long *rmapp
,
976 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
977 unsigned long head
, i
, j
;
981 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
985 if (*rmapp
& KVMPPC_RMAP_REFERENCED
)
988 if (*rmapp
& KVMPPC_RMAP_PRESENT
) {
989 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
991 hp
= (unsigned long *)(kvm
->arch
.hpt_virt
+ (i
<< 4));
993 if (hp
[1] & HPTE_R_R
)
995 } while ((i
= j
) != head
);
1004 int kvm_test_age_hva(struct kvm
*kvm
, unsigned long hva
)
1006 if (!kvm
->arch
.using_mmu_notifiers
)
1008 return kvm_handle_hva(kvm
, hva
, kvm_test_age_rmapp
);
1011 void kvm_set_spte_hva(struct kvm
*kvm
, unsigned long hva
, pte_t pte
)
1013 if (!kvm
->arch
.using_mmu_notifiers
)
1015 kvm_handle_hva(kvm
, hva
, kvm_unmap_rmapp
);
1018 static int kvm_test_clear_dirty(struct kvm
*kvm
, unsigned long *rmapp
)
1020 struct revmap_entry
*rev
= kvm
->arch
.revmap
;
1021 unsigned long head
, i
, j
;
1022 unsigned long *hptep
;
1027 if (*rmapp
& KVMPPC_RMAP_CHANGED
) {
1028 *rmapp
&= ~KVMPPC_RMAP_CHANGED
;
1031 if (!(*rmapp
& KVMPPC_RMAP_PRESENT
)) {
1036 i
= head
= *rmapp
& KVMPPC_RMAP_INDEX
;
1038 hptep
= (unsigned long *) (kvm
->arch
.hpt_virt
+ (i
<< 4));
1041 if (!(hptep
[1] & HPTE_R_C
))
1044 if (!try_lock_hpte(hptep
, HPTE_V_HVLOCK
)) {
1045 /* unlock rmap before spinning on the HPTE lock */
1047 while (hptep
[0] & HPTE_V_HVLOCK
)
1052 /* Now check and modify the HPTE */
1053 if ((hptep
[0] & HPTE_V_VALID
) && (hptep
[1] & HPTE_R_C
)) {
1054 /* need to make it temporarily absent to clear C */
1055 hptep
[0] |= HPTE_V_ABSENT
;
1056 kvmppc_invalidate_hpte(kvm
, hptep
, i
);
1057 hptep
[1] &= ~HPTE_R_C
;
1059 hptep
[0] = (hptep
[0] & ~HPTE_V_ABSENT
) | HPTE_V_VALID
;
1060 rev
[i
].guest_rpte
|= HPTE_R_C
;
1063 hptep
[0] &= ~HPTE_V_HVLOCK
;
1064 } while ((i
= j
) != head
);
1070 long kvmppc_hv_get_dirty_log(struct kvm
*kvm
, struct kvm_memory_slot
*memslot
,
1074 unsigned long *rmapp
;
1077 rmapp
= memslot
->arch
.rmap
;
1078 for (i
= 0; i
< memslot
->npages
; ++i
) {
1079 if (kvm_test_clear_dirty(kvm
, rmapp
) && map
)
1080 __set_bit_le(i
, map
);
1087 void *kvmppc_pin_guest_page(struct kvm
*kvm
, unsigned long gpa
,
1088 unsigned long *nb_ret
)
1090 struct kvm_memory_slot
*memslot
;
1091 unsigned long gfn
= gpa
>> PAGE_SHIFT
;
1092 struct page
*page
, *pages
[1];
1094 unsigned long hva
, psize
, offset
;
1096 unsigned long *physp
;
1099 srcu_idx
= srcu_read_lock(&kvm
->srcu
);
1100 memslot
= gfn_to_memslot(kvm
, gfn
);
1101 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
1103 if (!kvm
->arch
.using_mmu_notifiers
) {
1104 physp
= memslot
->arch
.slot_phys
;
1107 physp
+= gfn
- memslot
->base_gfn
;
1110 if (kvmppc_get_guest_page(kvm
, gfn
, memslot
,
1115 page
= pfn_to_page(pa
>> PAGE_SHIFT
);
1118 hva
= gfn_to_hva_memslot(memslot
, gfn
);
1119 npages
= get_user_pages_fast(hva
, 1, 1, pages
);
1124 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1127 if (PageHuge(page
)) {
1128 page
= compound_head(page
);
1129 psize
<<= compound_order(page
);
1131 offset
= gpa
& (psize
- 1);
1133 *nb_ret
= psize
- offset
;
1134 return page_address(page
) + offset
;
1137 srcu_read_unlock(&kvm
->srcu
, srcu_idx
);
1141 void kvmppc_unpin_guest_page(struct kvm
*kvm
, void *va
)
1143 struct page
*page
= virt_to_page(va
);
1148 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu
*vcpu
)
1150 struct kvmppc_mmu
*mmu
= &vcpu
->arch
.mmu
;
1152 if (cpu_has_feature(CPU_FTR_ARCH_206
))
1153 vcpu
->arch
.slb_nr
= 32; /* POWER7 */
1155 vcpu
->arch
.slb_nr
= 64;
1157 mmu
->xlate
= kvmppc_mmu_book3s_64_hv_xlate
;
1158 mmu
->reset_msr
= kvmppc_mmu_book3s_64_hv_reset_msr
;
1160 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SLB
;