2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
9 * Copyright (C) 2006 Qumranet, Inc.
10 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
13 * Yaniv Kamay <yaniv@qumranet.com>
14 * Avi Kivity <avi@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
22 * We need the mmu code to access both 32-bit and 64-bit guest ptes,
23 * so the code in this file is compiled twice, once per pte size.
27 #define pt_element_t u64
28 #define guest_walker guest_walker64
29 #define FNAME(name) paging##64_##name
30 #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
31 #define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
32 #define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
33 #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
34 #define PT_LEVEL_BITS PT64_LEVEL_BITS
36 #define PT_MAX_FULL_LEVELS 4
37 #define CMPXCHG cmpxchg
39 #define CMPXCHG cmpxchg64
40 #define PT_MAX_FULL_LEVELS 2
43 #define pt_element_t u32
44 #define guest_walker guest_walker32
45 #define FNAME(name) paging##32_##name
46 #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
47 #define PT_LVL_ADDR_MASK(lvl) PT32_LVL_ADDR_MASK(lvl)
48 #define PT_LVL_OFFSET_MASK(lvl) PT32_LVL_OFFSET_MASK(lvl)
49 #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
50 #define PT_LEVEL_BITS PT32_LEVEL_BITS
51 #define PT_MAX_FULL_LEVELS 2
52 #define CMPXCHG cmpxchg
54 #error Invalid PTTYPE value
57 #define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl)
58 #define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PT_PAGE_TABLE_LEVEL)
61 * The guest_walker structure emulates the behavior of the hardware page
66 gfn_t table_gfn
[PT_MAX_FULL_LEVELS
];
67 pt_element_t ptes
[PT_MAX_FULL_LEVELS
];
68 pt_element_t prefetch_ptes
[PTE_PREFETCH_NUM
];
69 gpa_t pte_gpa
[PT_MAX_FULL_LEVELS
];
73 struct x86_exception fault
;
76 static gfn_t
gpte_to_gfn_lvl(pt_element_t gpte
, int lvl
)
78 return (gpte
& PT_LVL_ADDR_MASK(lvl
)) >> PAGE_SHIFT
;
81 static int FNAME(cmpxchg_gpte
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
82 pt_element_t __user
*ptep_user
, unsigned index
,
83 pt_element_t orig_pte
, pt_element_t new_pte
)
90 npages
= get_user_pages_fast((unsigned long)ptep_user
, 1, 1, &page
);
91 /* Check if the user is doing something meaningless. */
92 if (unlikely(npages
!= 1))
95 table
= kmap_atomic(page
, KM_USER0
);
96 ret
= CMPXCHG(&table
[index
], orig_pte
, new_pte
);
97 kunmap_atomic(table
, KM_USER0
);
99 kvm_release_page_dirty(page
);
101 return (ret
!= orig_pte
);
104 static unsigned FNAME(gpte_access
)(struct kvm_vcpu
*vcpu
, pt_element_t gpte
)
108 access
= (gpte
& (PT_WRITABLE_MASK
| PT_USER_MASK
)) | ACC_EXEC_MASK
;
110 if (vcpu
->arch
.mmu
.nx
)
111 access
&= ~(gpte
>> PT64_NX_SHIFT
);
116 static bool FNAME(is_last_gpte
)(struct guest_walker
*walker
,
117 struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
120 if (walker
->level
== PT_PAGE_TABLE_LEVEL
)
123 if ((walker
->level
== PT_DIRECTORY_LEVEL
) && is_large_pte(gpte
) &&
124 (PTTYPE
== 64 || is_pse(vcpu
)))
127 if ((walker
->level
== PT_PDPE_LEVEL
) && is_large_pte(gpte
) &&
128 (mmu
->root_level
== PT64_ROOT_LEVEL
))
135 * Fetch a guest pte for a guest virtual address
137 static int FNAME(walk_addr_generic
)(struct guest_walker
*walker
,
138 struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
139 gva_t addr
, u32 access
)
142 pt_element_t __user
*uninitialized_var(ptep_user
);
144 unsigned index
, pt_access
, uninitialized_var(pte_access
);
148 const int write_fault
= access
& PFERR_WRITE_MASK
;
149 const int user_fault
= access
& PFERR_USER_MASK
;
150 const int fetch_fault
= access
& PFERR_FETCH_MASK
;
153 trace_kvm_mmu_pagetable_walk(addr
, write_fault
, user_fault
,
157 walker
->level
= mmu
->root_level
;
158 pte
= mmu
->get_cr3(vcpu
);
161 if (walker
->level
== PT32E_ROOT_LEVEL
) {
162 pte
= kvm_pdptr_read_mmu(vcpu
, mmu
, (addr
>> 30) & 3);
163 trace_kvm_mmu_paging_element(pte
, walker
->level
);
164 if (!is_present_gpte(pte
))
169 ASSERT((!is_long_mode(vcpu
) && is_pae(vcpu
)) ||
170 (mmu
->get_cr3(vcpu
) & CR3_NONPAE_RESERVED_BITS
) == 0);
176 unsigned long host_addr
;
178 index
= PT_INDEX(addr
, walker
->level
);
180 table_gfn
= gpte_to_gfn(pte
);
181 offset
= index
* sizeof(pt_element_t
);
182 pte_gpa
= gfn_to_gpa(table_gfn
) + offset
;
183 walker
->table_gfn
[walker
->level
- 1] = table_gfn
;
184 walker
->pte_gpa
[walker
->level
- 1] = pte_gpa
;
186 real_gfn
= mmu
->translate_gpa(vcpu
, gfn_to_gpa(table_gfn
),
187 PFERR_USER_MASK
|PFERR_WRITE_MASK
);
188 if (unlikely(real_gfn
== UNMAPPED_GVA
))
190 real_gfn
= gpa_to_gfn(real_gfn
);
192 host_addr
= gfn_to_hva(vcpu
->kvm
, real_gfn
);
193 if (unlikely(kvm_is_error_hva(host_addr
)))
196 ptep_user
= (pt_element_t __user
*)((void *)host_addr
+ offset
);
197 if (unlikely(__copy_from_user(&pte
, ptep_user
, sizeof(pte
))))
200 trace_kvm_mmu_paging_element(pte
, walker
->level
);
202 if (unlikely(!is_present_gpte(pte
)))
205 if (unlikely(is_rsvd_bits_set(&vcpu
->arch
.mmu
, pte
,
207 errcode
|= PFERR_RSVD_MASK
| PFERR_PRESENT_MASK
;
211 if (!check_write_user_access(vcpu
, write_fault
, user_fault
,
216 if (unlikely(fetch_fault
&& (pte
& PT64_NX_MASK
)))
220 if (!eperm
&& unlikely(!(pte
& PT_ACCESSED_MASK
))) {
222 trace_kvm_mmu_set_accessed_bit(table_gfn
, index
,
224 ret
= FNAME(cmpxchg_gpte
)(vcpu
, mmu
, ptep_user
, index
,
225 pte
, pte
|PT_ACCESSED_MASK
);
226 if (unlikely(ret
< 0))
231 mark_page_dirty(vcpu
->kvm
, table_gfn
);
232 pte
|= PT_ACCESSED_MASK
;
235 pte_access
= pt_access
& FNAME(gpte_access
)(vcpu
, pte
);
237 walker
->ptes
[walker
->level
- 1] = pte
;
239 if (FNAME(is_last_gpte
)(walker
, vcpu
, mmu
, pte
)) {
240 int lvl
= walker
->level
;
245 /* check if the kernel is fetching from user page */
246 if (unlikely(pte_access
& PT_USER_MASK
) &&
247 kvm_read_cr4_bits(vcpu
, X86_CR4_SMEP
))
248 if (fetch_fault
&& !user_fault
)
251 gfn
= gpte_to_gfn_lvl(pte
, lvl
);
252 gfn
+= (addr
& PT_LVL_OFFSET_MASK(lvl
)) >> PAGE_SHIFT
;
255 walker
->level
== PT_DIRECTORY_LEVEL
&&
257 gfn
+= pse36_gfn_delta(pte
);
259 ac
= write_fault
| fetch_fault
| user_fault
;
261 real_gpa
= mmu
->translate_gpa(vcpu
, gfn_to_gpa(gfn
),
263 if (real_gpa
== UNMAPPED_GVA
)
266 walker
->gfn
= real_gpa
>> PAGE_SHIFT
;
271 pt_access
= pte_access
;
275 if (unlikely(eperm
)) {
276 errcode
|= PFERR_PRESENT_MASK
;
280 if (write_fault
&& unlikely(!is_dirty_gpte(pte
))) {
283 trace_kvm_mmu_set_dirty_bit(table_gfn
, index
, sizeof(pte
));
284 ret
= FNAME(cmpxchg_gpte
)(vcpu
, mmu
, ptep_user
, index
,
285 pte
, pte
|PT_DIRTY_MASK
);
286 if (unlikely(ret
< 0))
291 mark_page_dirty(vcpu
->kvm
, table_gfn
);
292 pte
|= PT_DIRTY_MASK
;
293 walker
->ptes
[walker
->level
- 1] = pte
;
296 walker
->pt_access
= pt_access
;
297 walker
->pte_access
= pte_access
;
298 pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
299 __func__
, (u64
)pte
, pte_access
, pt_access
);
303 errcode
|= write_fault
| user_fault
;
304 if (fetch_fault
&& (mmu
->nx
||
305 kvm_read_cr4_bits(vcpu
, X86_CR4_SMEP
)))
306 errcode
|= PFERR_FETCH_MASK
;
308 walker
->fault
.vector
= PF_VECTOR
;
309 walker
->fault
.error_code_valid
= true;
310 walker
->fault
.error_code
= errcode
;
311 walker
->fault
.address
= addr
;
312 walker
->fault
.nested_page_fault
= mmu
!= vcpu
->arch
.walk_mmu
;
314 trace_kvm_mmu_walker_error(walker
->fault
.error_code
);
318 static int FNAME(walk_addr
)(struct guest_walker
*walker
,
319 struct kvm_vcpu
*vcpu
, gva_t addr
, u32 access
)
321 return FNAME(walk_addr_generic
)(walker
, vcpu
, &vcpu
->arch
.mmu
, addr
,
325 static int FNAME(walk_addr_nested
)(struct guest_walker
*walker
,
326 struct kvm_vcpu
*vcpu
, gva_t addr
,
329 return FNAME(walk_addr_generic
)(walker
, vcpu
, &vcpu
->arch
.nested_mmu
,
333 static bool FNAME(prefetch_invalid_gpte
)(struct kvm_vcpu
*vcpu
,
334 struct kvm_mmu_page
*sp
, u64
*spte
,
337 u64 nonpresent
= shadow_trap_nonpresent_pte
;
339 if (is_rsvd_bits_set(&vcpu
->arch
.mmu
, gpte
, PT_PAGE_TABLE_LEVEL
))
342 if (!is_present_gpte(gpte
)) {
344 nonpresent
= shadow_notrap_nonpresent_pte
;
348 if (!(gpte
& PT_ACCESSED_MASK
))
354 drop_spte(vcpu
->kvm
, spte
, nonpresent
);
358 static void FNAME(update_pte
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*sp
,
359 u64
*spte
, const void *pte
)
365 gpte
= *(const pt_element_t
*)pte
;
366 if (FNAME(prefetch_invalid_gpte
)(vcpu
, sp
, spte
, gpte
))
369 pgprintk("%s: gpte %llx spte %p\n", __func__
, (u64
)gpte
, spte
);
370 pte_access
= sp
->role
.access
& FNAME(gpte_access
)(vcpu
, gpte
);
371 pfn
= gfn_to_pfn_atomic(vcpu
->kvm
, gpte_to_gfn(gpte
));
372 if (is_error_pfn(pfn
)) {
373 kvm_release_pfn_clean(pfn
);
378 * we call mmu_set_spte() with host_writable = true because that
379 * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1).
381 mmu_set_spte(vcpu
, spte
, sp
->role
.access
, pte_access
, 0, 0,
382 is_dirty_gpte(gpte
), NULL
, PT_PAGE_TABLE_LEVEL
,
383 gpte_to_gfn(gpte
), pfn
, true, true);
386 static bool FNAME(gpte_changed
)(struct kvm_vcpu
*vcpu
,
387 struct guest_walker
*gw
, int level
)
389 pt_element_t curr_pte
;
390 gpa_t base_gpa
, pte_gpa
= gw
->pte_gpa
[level
- 1];
394 if (level
== PT_PAGE_TABLE_LEVEL
) {
395 mask
= PTE_PREFETCH_NUM
* sizeof(pt_element_t
) - 1;
396 base_gpa
= pte_gpa
& ~mask
;
397 index
= (pte_gpa
- base_gpa
) / sizeof(pt_element_t
);
399 r
= kvm_read_guest_atomic(vcpu
->kvm
, base_gpa
,
400 gw
->prefetch_ptes
, sizeof(gw
->prefetch_ptes
));
401 curr_pte
= gw
->prefetch_ptes
[index
];
403 r
= kvm_read_guest_atomic(vcpu
->kvm
, pte_gpa
,
404 &curr_pte
, sizeof(curr_pte
));
406 return r
|| curr_pte
!= gw
->ptes
[level
- 1];
409 static void FNAME(pte_prefetch
)(struct kvm_vcpu
*vcpu
, struct guest_walker
*gw
,
412 struct kvm_mmu_page
*sp
;
413 pt_element_t
*gptep
= gw
->prefetch_ptes
;
417 sp
= page_header(__pa(sptep
));
419 if (sp
->role
.level
> PT_PAGE_TABLE_LEVEL
)
423 return __direct_pte_prefetch(vcpu
, sp
, sptep
);
425 i
= (sptep
- sp
->spt
) & ~(PTE_PREFETCH_NUM
- 1);
428 for (i
= 0; i
< PTE_PREFETCH_NUM
; i
++, spte
++) {
438 if (*spte
!= shadow_trap_nonpresent_pte
)
443 if (FNAME(prefetch_invalid_gpte
)(vcpu
, sp
, spte
, gpte
))
446 pte_access
= sp
->role
.access
& FNAME(gpte_access
)(vcpu
, gpte
);
447 gfn
= gpte_to_gfn(gpte
);
448 dirty
= is_dirty_gpte(gpte
);
449 pfn
= pte_prefetch_gfn_to_pfn(vcpu
, gfn
,
450 (pte_access
& ACC_WRITE_MASK
) && dirty
);
451 if (is_error_pfn(pfn
)) {
452 kvm_release_pfn_clean(pfn
);
456 mmu_set_spte(vcpu
, spte
, sp
->role
.access
, pte_access
, 0, 0,
457 dirty
, NULL
, PT_PAGE_TABLE_LEVEL
, gfn
,
463 * Fetch a shadow pte for a specific level in the paging hierarchy.
465 static u64
*FNAME(fetch
)(struct kvm_vcpu
*vcpu
, gva_t addr
,
466 struct guest_walker
*gw
,
467 int user_fault
, int write_fault
, int hlevel
,
468 int *ptwrite
, pfn_t pfn
, bool map_writable
,
471 unsigned access
= gw
->pt_access
;
472 struct kvm_mmu_page
*sp
= NULL
;
473 bool dirty
= is_dirty_gpte(gw
->ptes
[gw
->level
- 1]);
475 unsigned direct_access
;
476 struct kvm_shadow_walk_iterator it
;
478 if (!is_present_gpte(gw
->ptes
[gw
->level
- 1]))
481 direct_access
= gw
->pt_access
& gw
->pte_access
;
483 direct_access
&= ~ACC_WRITE_MASK
;
485 top_level
= vcpu
->arch
.mmu
.root_level
;
486 if (top_level
== PT32E_ROOT_LEVEL
)
487 top_level
= PT32_ROOT_LEVEL
;
489 * Verify that the top-level gpte is still there. Since the page
490 * is a root page, it is either write protected (and cannot be
491 * changed from now on) or it is invalid (in which case, we don't
492 * really care if it changes underneath us after this point).
494 if (FNAME(gpte_changed
)(vcpu
, gw
, top_level
))
495 goto out_gpte_changed
;
497 for (shadow_walk_init(&it
, vcpu
, addr
);
498 shadow_walk_okay(&it
) && it
.level
> gw
->level
;
499 shadow_walk_next(&it
)) {
502 drop_large_spte(vcpu
, it
.sptep
);
505 if (!is_shadow_present_pte(*it
.sptep
)) {
506 table_gfn
= gw
->table_gfn
[it
.level
- 2];
507 sp
= kvm_mmu_get_page(vcpu
, table_gfn
, addr
, it
.level
-1,
508 false, access
, it
.sptep
);
512 * Verify that the gpte in the page we've just write
513 * protected is still there.
515 if (FNAME(gpte_changed
)(vcpu
, gw
, it
.level
- 1))
516 goto out_gpte_changed
;
519 link_shadow_page(it
.sptep
, sp
);
523 shadow_walk_okay(&it
) && it
.level
> hlevel
;
524 shadow_walk_next(&it
)) {
527 validate_direct_spte(vcpu
, it
.sptep
, direct_access
);
529 drop_large_spte(vcpu
, it
.sptep
);
531 if (is_shadow_present_pte(*it
.sptep
))
534 direct_gfn
= gw
->gfn
& ~(KVM_PAGES_PER_HPAGE(it
.level
) - 1);
536 sp
= kvm_mmu_get_page(vcpu
, direct_gfn
, addr
, it
.level
-1,
537 true, direct_access
, it
.sptep
);
538 link_shadow_page(it
.sptep
, sp
);
541 mmu_set_spte(vcpu
, it
.sptep
, access
, gw
->pte_access
& access
,
542 user_fault
, write_fault
, dirty
, ptwrite
, it
.level
,
543 gw
->gfn
, pfn
, prefault
, map_writable
);
544 FNAME(pte_prefetch
)(vcpu
, gw
, it
.sptep
);
550 kvm_mmu_put_page(sp
, it
.sptep
);
551 kvm_release_pfn_clean(pfn
);
556 * Page fault handler. There are several causes for a page fault:
557 * - there is no shadow pte for the guest pte
558 * - write access through a shadow pte marked read only so that we can set
560 * - write access to a shadow pte marked read only so we can update the page
561 * dirty bitmap, when userspace requests it
562 * - mmio access; in this case we will never install a present shadow pte
563 * - normal guest page fault due to the guest pte marked not present, not
564 * writable, or not executable
566 * Returns: 1 if we need to emulate the instruction, 0 otherwise, or
567 * a negative value on error.
569 static int FNAME(page_fault
)(struct kvm_vcpu
*vcpu
, gva_t addr
, u32 error_code
,
572 int write_fault
= error_code
& PFERR_WRITE_MASK
;
573 int user_fault
= error_code
& PFERR_USER_MASK
;
574 struct guest_walker walker
;
579 int level
= PT_PAGE_TABLE_LEVEL
;
581 unsigned long mmu_seq
;
584 pgprintk("%s: addr %lx err %x\n", __func__
, addr
, error_code
);
586 r
= mmu_topup_memory_caches(vcpu
);
591 * Look up the guest pte for the faulting address.
593 r
= FNAME(walk_addr
)(&walker
, vcpu
, addr
, error_code
);
596 * The page is not mapped by the guest. Let the guest handle it.
599 pgprintk("%s: guest page fault\n", __func__
);
601 inject_page_fault(vcpu
, &walker
.fault
);
602 /* reset fork detector */
603 vcpu
->arch
.last_pt_write_count
= 0;
608 if (walker
.level
>= PT_DIRECTORY_LEVEL
)
609 force_pt_level
= mapping_level_dirty_bitmap(vcpu
, walker
.gfn
);
612 if (!force_pt_level
) {
613 level
= min(walker
.level
, mapping_level(vcpu
, walker
.gfn
));
614 walker
.gfn
= walker
.gfn
& ~(KVM_PAGES_PER_HPAGE(level
) - 1);
617 mmu_seq
= vcpu
->kvm
->mmu_notifier_seq
;
620 if (try_async_pf(vcpu
, prefault
, walker
.gfn
, addr
, &pfn
, write_fault
,
625 if (is_error_pfn(pfn
)) {
626 unsigned access
= walker
.pte_access
;
627 bool dirty
= is_dirty_gpte(walker
.ptes
[walker
.level
- 1]);
630 access
&= ~ACC_WRITE_MASK
;
632 return kvm_handle_bad_page(vcpu
, mmu_is_nested(vcpu
) ? 0 :
633 addr
, access
, walker
.gfn
, pfn
);
636 spin_lock(&vcpu
->kvm
->mmu_lock
);
637 if (mmu_notifier_retry(vcpu
, mmu_seq
))
640 trace_kvm_mmu_audit(vcpu
, AUDIT_PRE_PAGE_FAULT
);
641 kvm_mmu_free_some_pages(vcpu
);
643 transparent_hugepage_adjust(vcpu
, &walker
.gfn
, &pfn
, &level
);
644 sptep
= FNAME(fetch
)(vcpu
, addr
, &walker
, user_fault
, write_fault
,
645 level
, &write_pt
, pfn
, map_writable
, prefault
);
647 pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__
,
648 sptep
, *sptep
, write_pt
);
651 vcpu
->arch
.last_pt_write_count
= 0; /* reset fork detector */
653 ++vcpu
->stat
.pf_fixed
;
654 trace_kvm_mmu_audit(vcpu
, AUDIT_POST_PAGE_FAULT
);
655 spin_unlock(&vcpu
->kvm
->mmu_lock
);
660 spin_unlock(&vcpu
->kvm
->mmu_lock
);
661 kvm_release_pfn_clean(pfn
);
665 static void FNAME(invlpg
)(struct kvm_vcpu
*vcpu
, gva_t gva
)
667 struct kvm_shadow_walk_iterator iterator
;
668 struct kvm_mmu_page
*sp
;
674 vcpu_clear_mmio_info(vcpu
, gva
);
676 spin_lock(&vcpu
->kvm
->mmu_lock
);
678 for_each_shadow_entry(vcpu
, gva
, iterator
) {
679 level
= iterator
.level
;
680 sptep
= iterator
.sptep
;
682 sp
= page_header(__pa(sptep
));
683 if (is_last_spte(*sptep
, level
)) {
690 (PT_LEVEL_BITS
- PT64_LEVEL_BITS
) * level
;
691 offset
= sp
->role
.quadrant
<< shift
;
693 pte_gpa
= (sp
->gfn
<< PAGE_SHIFT
) + offset
;
694 pte_gpa
+= (sptep
- sp
->spt
) * sizeof(pt_element_t
);
696 if (is_shadow_present_pte(*sptep
)) {
697 if (is_large_pte(*sptep
))
698 --vcpu
->kvm
->stat
.lpages
;
699 drop_spte(vcpu
->kvm
, sptep
,
700 shadow_trap_nonpresent_pte
);
703 __set_spte(sptep
, shadow_trap_nonpresent_pte
);
707 if (!is_shadow_present_pte(*sptep
) || !sp
->unsync_children
)
712 kvm_flush_remote_tlbs(vcpu
->kvm
);
714 atomic_inc(&vcpu
->kvm
->arch
.invlpg_counter
);
716 spin_unlock(&vcpu
->kvm
->mmu_lock
);
721 if (mmu_topup_memory_caches(vcpu
))
723 kvm_mmu_pte_write(vcpu
, pte_gpa
, NULL
, sizeof(pt_element_t
), 0);
726 static gpa_t
FNAME(gva_to_gpa
)(struct kvm_vcpu
*vcpu
, gva_t vaddr
, u32 access
,
727 struct x86_exception
*exception
)
729 struct guest_walker walker
;
730 gpa_t gpa
= UNMAPPED_GVA
;
733 r
= FNAME(walk_addr
)(&walker
, vcpu
, vaddr
, access
);
736 gpa
= gfn_to_gpa(walker
.gfn
);
737 gpa
|= vaddr
& ~PAGE_MASK
;
738 } else if (exception
)
739 *exception
= walker
.fault
;
744 static gpa_t
FNAME(gva_to_gpa_nested
)(struct kvm_vcpu
*vcpu
, gva_t vaddr
,
746 struct x86_exception
*exception
)
748 struct guest_walker walker
;
749 gpa_t gpa
= UNMAPPED_GVA
;
752 r
= FNAME(walk_addr_nested
)(&walker
, vcpu
, vaddr
, access
);
755 gpa
= gfn_to_gpa(walker
.gfn
);
756 gpa
|= vaddr
& ~PAGE_MASK
;
757 } else if (exception
)
758 *exception
= walker
.fault
;
763 static void FNAME(prefetch_page
)(struct kvm_vcpu
*vcpu
,
764 struct kvm_mmu_page
*sp
)
767 pt_element_t pt
[256 / sizeof(pt_element_t
)];
771 || (PTTYPE
== 32 && sp
->role
.level
> PT_PAGE_TABLE_LEVEL
)) {
772 nonpaging_prefetch_page(vcpu
, sp
);
776 pte_gpa
= gfn_to_gpa(sp
->gfn
);
778 offset
= sp
->role
.quadrant
<< PT64_LEVEL_BITS
;
779 pte_gpa
+= offset
* sizeof(pt_element_t
);
782 for (i
= 0; i
< PT64_ENT_PER_PAGE
; i
+= ARRAY_SIZE(pt
)) {
783 r
= kvm_read_guest_atomic(vcpu
->kvm
, pte_gpa
, pt
, sizeof pt
);
784 pte_gpa
+= ARRAY_SIZE(pt
) * sizeof(pt_element_t
);
785 for (j
= 0; j
< ARRAY_SIZE(pt
); ++j
)
786 if (r
|| is_present_gpte(pt
[j
]))
787 sp
->spt
[i
+j
] = shadow_trap_nonpresent_pte
;
789 sp
->spt
[i
+j
] = shadow_notrap_nonpresent_pte
;
794 * Using the cached information from sp->gfns is safe because:
795 * - The spte has a reference to the struct page, so the pfn for a given gfn
796 * can't change unless all sptes pointing to it are nuked first.
799 * We should flush all tlbs if spte is dropped even though guest is
800 * responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page
801 * and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't
802 * used by guest then tlbs are not flushed, so guest is allowed to access the
804 * And we increase kvm->tlbs_dirty to delay tlbs flush in this case.
806 static int FNAME(sync_page
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*sp
)
808 int i
, offset
, nr_present
;
812 offset
= nr_present
= 0;
814 /* direct kvm_mmu_page can not be unsync. */
815 BUG_ON(sp
->role
.direct
);
818 offset
= sp
->role
.quadrant
<< PT64_LEVEL_BITS
;
820 first_pte_gpa
= gfn_to_gpa(sp
->gfn
) + offset
* sizeof(pt_element_t
);
822 for (i
= 0; i
< PT64_ENT_PER_PAGE
; i
++) {
828 if (!is_shadow_present_pte(sp
->spt
[i
]))
831 pte_gpa
= first_pte_gpa
+ i
* sizeof(pt_element_t
);
833 if (kvm_read_guest_atomic(vcpu
->kvm
, pte_gpa
, &gpte
,
834 sizeof(pt_element_t
)))
837 gfn
= gpte_to_gfn(gpte
);
839 if (FNAME(prefetch_invalid_gpte
)(vcpu
, sp
, &sp
->spt
[i
], gpte
)) {
840 vcpu
->kvm
->tlbs_dirty
++;
844 if (gfn
!= sp
->gfns
[i
]) {
845 drop_spte(vcpu
->kvm
, &sp
->spt
[i
],
846 shadow_trap_nonpresent_pte
);
847 vcpu
->kvm
->tlbs_dirty
++;
852 pte_access
= sp
->role
.access
& FNAME(gpte_access
)(vcpu
, gpte
);
853 host_writable
= sp
->spt
[i
] & SPTE_HOST_WRITEABLE
;
855 set_spte(vcpu
, &sp
->spt
[i
], pte_access
, 0, 0,
856 is_dirty_gpte(gpte
), PT_PAGE_TABLE_LEVEL
, gfn
,
857 spte_to_pfn(sp
->spt
[i
]), true, false,
867 #undef PT_BASE_ADDR_MASK
869 #undef PT_LVL_ADDR_MASK
870 #undef PT_LVL_OFFSET_MASK
872 #undef PT_MAX_FULL_LEVELS
874 #undef gpte_to_gfn_lvl