2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
9 * Copyright (C) 2006 Qumranet, Inc.
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
21 * We need the mmu code to access both 32-bit and 64-bit guest ptes,
22 * so the code in this file is compiled twice, once per pte size.
26 #define pt_element_t u64
27 #define guest_walker guest_walker64
28 #define FNAME(name) paging##64_##name
29 #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
30 #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK
31 #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
32 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
33 #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level)
34 #define PT_LEVEL_BITS PT64_LEVEL_BITS
36 #define PT_MAX_FULL_LEVELS 4
37 #define CMPXCHG cmpxchg
39 #define CMPXCHG cmpxchg64
40 #define PT_MAX_FULL_LEVELS 2
43 #define pt_element_t u32
44 #define guest_walker guest_walker32
45 #define FNAME(name) paging##32_##name
46 #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
47 #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK
48 #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
49 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
50 #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level)
51 #define PT_LEVEL_BITS PT32_LEVEL_BITS
52 #define PT_MAX_FULL_LEVELS 2
53 #define CMPXCHG cmpxchg
55 #error Invalid PTTYPE value
58 #define gpte_to_gfn FNAME(gpte_to_gfn)
59 #define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde)
62 * The guest_walker structure emulates the behavior of the hardware page
67 gfn_t table_gfn
[PT_MAX_FULL_LEVELS
];
68 pt_element_t ptes
[PT_MAX_FULL_LEVELS
];
69 gpa_t pte_gpa
[PT_MAX_FULL_LEVELS
];
76 static gfn_t
gpte_to_gfn(pt_element_t gpte
)
78 return (gpte
& PT_BASE_ADDR_MASK
) >> PAGE_SHIFT
;
81 static gfn_t
gpte_to_gfn_pde(pt_element_t gpte
)
83 return (gpte
& PT_DIR_BASE_ADDR_MASK
) >> PAGE_SHIFT
;
86 static bool FNAME(cmpxchg_gpte
)(struct kvm
*kvm
,
87 gfn_t table_gfn
, unsigned index
,
88 pt_element_t orig_pte
, pt_element_t new_pte
)
94 page
= gfn_to_page(kvm
, table_gfn
);
95 table
= kmap_atomic(page
, KM_USER0
);
97 ret
= CMPXCHG(&table
[index
], orig_pte
, new_pte
);
99 kunmap_atomic(table
, KM_USER0
);
101 kvm_release_page_dirty(page
);
103 return (ret
!= orig_pte
);
106 static unsigned FNAME(gpte_access
)(struct kvm_vcpu
*vcpu
, pt_element_t gpte
)
110 access
= (gpte
& (PT_WRITABLE_MASK
| PT_USER_MASK
)) | ACC_EXEC_MASK
;
113 access
&= ~(gpte
>> PT64_NX_SHIFT
);
119 * Fetch a guest pte for a guest virtual address
121 static int FNAME(walk_addr
)(struct guest_walker
*walker
,
122 struct kvm_vcpu
*vcpu
, gva_t addr
,
123 int write_fault
, int user_fault
, int fetch_fault
)
127 unsigned index
, pt_access
, pte_access
;
130 pgprintk("%s: addr %lx\n", __FUNCTION__
, addr
);
132 walker
->level
= vcpu
->arch
.mmu
.root_level
;
133 pte
= vcpu
->arch
.cr3
;
135 if (!is_long_mode(vcpu
)) {
136 pte
= vcpu
->arch
.pdptrs
[(addr
>> 30) & 3];
137 if (!is_present_pte(pte
))
142 ASSERT((!is_long_mode(vcpu
) && is_pae(vcpu
)) ||
143 (vcpu
->cr3
& CR3_NONPAE_RESERVED_BITS
) == 0);
148 index
= PT_INDEX(addr
, walker
->level
);
150 table_gfn
= gpte_to_gfn(pte
);
151 pte_gpa
= gfn_to_gpa(table_gfn
);
152 pte_gpa
+= index
* sizeof(pt_element_t
);
153 walker
->table_gfn
[walker
->level
- 1] = table_gfn
;
154 walker
->pte_gpa
[walker
->level
- 1] = pte_gpa
;
155 pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__
,
156 walker
->level
- 1, table_gfn
);
158 kvm_read_guest(vcpu
->kvm
, pte_gpa
, &pte
, sizeof(pte
));
160 if (!is_present_pte(pte
))
163 if (write_fault
&& !is_writeble_pte(pte
))
164 if (user_fault
|| is_write_protection(vcpu
))
167 if (user_fault
&& !(pte
& PT_USER_MASK
))
171 if (fetch_fault
&& is_nx(vcpu
) && (pte
& PT64_NX_MASK
))
175 if (!(pte
& PT_ACCESSED_MASK
)) {
176 mark_page_dirty(vcpu
->kvm
, table_gfn
);
177 if (FNAME(cmpxchg_gpte
)(vcpu
->kvm
, table_gfn
,
178 index
, pte
, pte
|PT_ACCESSED_MASK
))
180 pte
|= PT_ACCESSED_MASK
;
183 pte_access
= pt_access
& FNAME(gpte_access
)(vcpu
, pte
);
185 walker
->ptes
[walker
->level
- 1] = pte
;
187 if (walker
->level
== PT_PAGE_TABLE_LEVEL
) {
188 walker
->gfn
= gpte_to_gfn(pte
);
192 if (walker
->level
== PT_DIRECTORY_LEVEL
193 && (pte
& PT_PAGE_SIZE_MASK
)
194 && (PTTYPE
== 64 || is_pse(vcpu
))) {
195 walker
->gfn
= gpte_to_gfn_pde(pte
);
196 walker
->gfn
+= PT_INDEX(addr
, PT_PAGE_TABLE_LEVEL
);
197 if (PTTYPE
== 32 && is_cpuid_PSE36())
198 walker
->gfn
+= pse36_gfn_delta(pte
);
202 pt_access
= pte_access
;
206 if (write_fault
&& !is_dirty_pte(pte
)) {
209 mark_page_dirty(vcpu
->kvm
, table_gfn
);
210 ret
= FNAME(cmpxchg_gpte
)(vcpu
->kvm
, table_gfn
, index
, pte
,
214 pte
|= PT_DIRTY_MASK
;
215 kvm_mmu_pte_write(vcpu
, pte_gpa
, (u8
*)&pte
, sizeof(pte
));
216 walker
->ptes
[walker
->level
- 1] = pte
;
219 walker
->pt_access
= pt_access
;
220 walker
->pte_access
= pte_access
;
221 pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
222 __FUNCTION__
, (u64
)pte
, pt_access
, pte_access
);
226 walker
->error_code
= 0;
230 walker
->error_code
= PFERR_PRESENT_MASK
;
234 walker
->error_code
|= PFERR_WRITE_MASK
;
236 walker
->error_code
|= PFERR_USER_MASK
;
238 walker
->error_code
|= PFERR_FETCH_MASK
;
242 static void FNAME(update_pte
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*page
,
243 u64
*spte
, const void *pte
, int bytes
,
249 gpte
= *(const pt_element_t
*)pte
;
250 if (~gpte
& (PT_PRESENT_MASK
| PT_ACCESSED_MASK
)) {
251 if (!offset_in_pte
&& !is_present_pte(gpte
))
252 set_shadow_pte(spte
, shadow_notrap_nonpresent_pte
);
255 if (bytes
< sizeof(pt_element_t
))
257 pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__
, (u64
)gpte
, spte
);
258 pte_access
= page
->role
.access
& FNAME(gpte_access
)(vcpu
, gpte
);
259 mmu_set_spte(vcpu
, spte
, page
->role
.access
, pte_access
, 0, 0,
260 gpte
& PT_DIRTY_MASK
, NULL
, gpte_to_gfn(gpte
));
264 * Fetch a shadow pte for a specific level in the paging hierarchy.
266 static u64
*FNAME(fetch
)(struct kvm_vcpu
*vcpu
, gva_t addr
,
267 struct guest_walker
*walker
,
268 int user_fault
, int write_fault
, int *ptwrite
)
273 unsigned access
= walker
->pt_access
;
275 if (!is_present_pte(walker
->ptes
[walker
->level
- 1]))
278 shadow_addr
= vcpu
->arch
.mmu
.root_hpa
;
279 level
= vcpu
->arch
.mmu
.shadow_root_level
;
280 if (level
== PT32E_ROOT_LEVEL
) {
281 shadow_addr
= vcpu
->arch
.mmu
.pae_root
[(addr
>> 30) & 3];
282 shadow_addr
&= PT64_BASE_ADDR_MASK
;
287 u32 index
= SHADOW_PT_INDEX(addr
, level
);
288 struct kvm_mmu_page
*shadow_page
;
294 shadow_ent
= ((u64
*)__va(shadow_addr
)) + index
;
295 if (is_shadow_present_pte(*shadow_ent
)) {
296 if (level
== PT_PAGE_TABLE_LEVEL
)
298 shadow_addr
= *shadow_ent
& PT64_BASE_ADDR_MASK
;
302 if (level
== PT_PAGE_TABLE_LEVEL
)
305 if (level
- 1 == PT_PAGE_TABLE_LEVEL
306 && walker
->level
== PT_DIRECTORY_LEVEL
) {
308 if (!is_dirty_pte(walker
->ptes
[level
- 1]))
309 access
&= ~ACC_WRITE_MASK
;
310 table_gfn
= gpte_to_gfn(walker
->ptes
[level
- 1]);
313 table_gfn
= walker
->table_gfn
[level
- 2];
315 shadow_page
= kvm_mmu_get_page(vcpu
, table_gfn
, addr
, level
-1,
316 metaphysical
, access
,
317 shadow_ent
, &new_page
);
318 if (new_page
&& !metaphysical
) {
320 pt_element_t curr_pte
;
321 r
= kvm_read_guest_atomic(vcpu
->kvm
,
322 walker
->pte_gpa
[level
- 2],
323 &curr_pte
, sizeof(curr_pte
));
324 if (r
|| curr_pte
!= walker
->ptes
[level
- 2])
327 shadow_addr
= __pa(shadow_page
->spt
);
328 shadow_pte
= shadow_addr
| PT_PRESENT_MASK
| PT_ACCESSED_MASK
329 | PT_WRITABLE_MASK
| PT_USER_MASK
;
330 *shadow_ent
= shadow_pte
;
333 mmu_set_spte(vcpu
, shadow_ent
, access
, walker
->pte_access
& access
,
334 user_fault
, write_fault
,
335 walker
->ptes
[walker
->level
-1] & PT_DIRTY_MASK
,
336 ptwrite
, walker
->gfn
);
342 * Page fault handler. There are several causes for a page fault:
343 * - there is no shadow pte for the guest pte
344 * - write access through a shadow pte marked read only so that we can set
346 * - write access to a shadow pte marked read only so we can update the page
347 * dirty bitmap, when userspace requests it
348 * - mmio access; in this case we will never install a present shadow pte
349 * - normal guest page fault due to the guest pte marked not present, not
350 * writable, or not executable
352 * Returns: 1 if we need to emulate the instruction, 0 otherwise, or
353 * a negative value on error.
355 static int FNAME(page_fault
)(struct kvm_vcpu
*vcpu
, gva_t addr
,
358 int write_fault
= error_code
& PFERR_WRITE_MASK
;
359 int user_fault
= error_code
& PFERR_USER_MASK
;
360 int fetch_fault
= error_code
& PFERR_FETCH_MASK
;
361 struct guest_walker walker
;
366 pgprintk("%s: addr %lx err %x\n", __FUNCTION__
, addr
, error_code
);
367 kvm_mmu_audit(vcpu
, "pre page fault");
369 r
= mmu_topup_memory_caches(vcpu
);
373 down_read(¤t
->mm
->mmap_sem
);
375 * Look up the shadow pte for the faulting address.
377 r
= FNAME(walk_addr
)(&walker
, vcpu
, addr
, write_fault
, user_fault
,
379 up_read(¤t
->mm
->mmap_sem
);
382 * The page is not mapped by the guest. Let the guest handle it.
385 pgprintk("%s: guest page fault\n", __FUNCTION__
);
386 inject_page_fault(vcpu
, addr
, walker
.error_code
);
387 vcpu
->arch
.last_pt_write_count
= 0; /* reset fork detector */
391 mutex_lock(&vcpu
->kvm
->lock
);
392 shadow_pte
= FNAME(fetch
)(vcpu
, addr
, &walker
, user_fault
, write_fault
,
394 pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__
,
395 shadow_pte
, *shadow_pte
, write_pt
);
398 vcpu
->arch
.last_pt_write_count
= 0; /* reset fork detector */
401 * mmio: emulate if accessible, otherwise its a guest fault.
403 if (shadow_pte
&& is_io_pte(*shadow_pte
)) {
404 mutex_unlock(&vcpu
->kvm
->lock
);
408 ++vcpu
->stat
.pf_fixed
;
409 kvm_mmu_audit(vcpu
, "post page fault (fixed)");
410 mutex_unlock(&vcpu
->kvm
->lock
);
415 static gpa_t
FNAME(gva_to_gpa
)(struct kvm_vcpu
*vcpu
, gva_t vaddr
)
417 struct guest_walker walker
;
418 gpa_t gpa
= UNMAPPED_GVA
;
421 r
= FNAME(walk_addr
)(&walker
, vcpu
, vaddr
, 0, 0, 0);
424 gpa
= gfn_to_gpa(walker
.gfn
);
425 gpa
|= vaddr
& ~PAGE_MASK
;
431 static void FNAME(prefetch_page
)(struct kvm_vcpu
*vcpu
,
432 struct kvm_mmu_page
*sp
)
434 int i
, offset
= 0, r
= 0;
437 if (sp
->role
.metaphysical
438 || (PTTYPE
== 32 && sp
->role
.level
> PT_PAGE_TABLE_LEVEL
)) {
439 nonpaging_prefetch_page(vcpu
, sp
);
444 offset
= sp
->role
.quadrant
<< PT64_LEVEL_BITS
;
446 for (i
= 0; i
< PT64_ENT_PER_PAGE
; ++i
) {
447 gpa_t pte_gpa
= gfn_to_gpa(sp
->gfn
);
448 pte_gpa
+= (i
+offset
) * sizeof(pt_element_t
);
450 r
= kvm_read_guest_atomic(vcpu
->kvm
, pte_gpa
, &pt
,
451 sizeof(pt_element_t
));
452 if (r
|| is_present_pte(pt
))
453 sp
->spt
[i
] = shadow_trap_nonpresent_pte
;
455 sp
->spt
[i
] = shadow_notrap_nonpresent_pte
;
462 #undef PT_BASE_ADDR_MASK
464 #undef SHADOW_PT_INDEX
466 #undef PT_DIR_BASE_ADDR_MASK
468 #undef PT_MAX_FULL_LEVELS
470 #undef gpte_to_gfn_pde