projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
KVM: MMU: Partial swapping of guest memory
[deliverable/linux.git] / drivers / kvm / mmu.c
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19
20 #include "vmx.h"
21 #include "kvm.h"
22
23 #include <linux/types.h>
24 #include <linux/string.h>
25 #include <linux/mm.h>
26 #include <linux/highmem.h>
27 #include <linux/module.h>
28
29 #include <asm/page.h>
30 #include <asm/cmpxchg.h>
31
32 #undef MMU_DEBUG
33
34 #undef AUDIT
35
36 #ifdef AUDIT
37 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
38 #else
39 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
40 #endif
41
42 #ifdef MMU_DEBUG
43
44 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
45 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
46
47 #else
48
49 #define pgprintk(x...) do { } while (0)
50 #define rmap_printk(x...) do { } while (0)
51
52 #endif
53
54 #if defined(MMU_DEBUG) || defined(AUDIT)
55 static int dbg = 1;
56 #endif
57
58 #ifndef MMU_DEBUG
59 #define ASSERT(x) do { } while (0)
60 #else
61 #define ASSERT(x) \
62 if (!(x)) { \
63 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
64 __FILE__, __LINE__, #x); \
65 }
66 #endif
67
68 #define PT64_PT_BITS 9
69 #define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
70 #define PT32_PT_BITS 10
71 #define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
72
73 #define PT_WRITABLE_SHIFT 1
74
75 #define PT_PRESENT_MASK (1ULL << 0)
76 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
77 #define PT_USER_MASK (1ULL << 2)
78 #define PT_PWT_MASK (1ULL << 3)
79 #define PT_PCD_MASK (1ULL << 4)
80 #define PT_ACCESSED_MASK (1ULL << 5)
81 #define PT_DIRTY_MASK (1ULL << 6)
82 #define PT_PAGE_SIZE_MASK (1ULL << 7)
83 #define PT_PAT_MASK (1ULL << 7)
84 #define PT_GLOBAL_MASK (1ULL << 8)
85 #define PT64_NX_MASK (1ULL << 63)
86
87 #define PT_PAT_SHIFT 7
88 #define PT_DIR_PAT_SHIFT 12
89 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
90
91 #define PT32_DIR_PSE36_SIZE 4
92 #define PT32_DIR_PSE36_SHIFT 13
93 #define PT32_DIR_PSE36_MASK \
94 (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
95
96
97 #define PT_FIRST_AVAIL_BITS_SHIFT 9
98 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
99
100 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
101
102 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
103
104 #define PT64_LEVEL_BITS 9
105
106 #define PT64_LEVEL_SHIFT(level) \
107 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
108
109 #define PT64_LEVEL_MASK(level) \
110 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
111
112 #define PT64_INDEX(address, level)\
113 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
114
115
116 #define PT32_LEVEL_BITS 10
117
118 #define PT32_LEVEL_SHIFT(level) \
119 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
120
121 #define PT32_LEVEL_MASK(level) \
122 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
123
124 #define PT32_INDEX(address, level)\
125 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
126
127
128 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
129 #define PT64_DIR_BASE_ADDR_MASK \
130 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
131
132 #define PT32_BASE_ADDR_MASK PAGE_MASK
133 #define PT32_DIR_BASE_ADDR_MASK \
134 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
135
136
137 #define PFERR_PRESENT_MASK (1U << 0)
138 #define PFERR_WRITE_MASK (1U << 1)
139 #define PFERR_USER_MASK (1U << 2)
140 #define PFERR_FETCH_MASK (1U << 4)
141
142 #define PT64_ROOT_LEVEL 4
143 #define PT32_ROOT_LEVEL 2
144 #define PT32E_ROOT_LEVEL 3
145
146 #define PT_DIRECTORY_LEVEL 2
147 #define PT_PAGE_TABLE_LEVEL 1
148
149 #define RMAP_EXT 4
150
151 struct kvm_rmap_desc {
152 u64 *shadow_ptes[RMAP_EXT];
153 struct kvm_rmap_desc *more;
154 };
155
156 static struct kmem_cache *pte_chain_cache;
157 static struct kmem_cache *rmap_desc_cache;
158 static struct kmem_cache *mmu_page_header_cache;
159
160 static u64 __read_mostly shadow_trap_nonpresent_pte;
161 static u64 __read_mostly shadow_notrap_nonpresent_pte;
162
163 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
164 {
165 shadow_trap_nonpresent_pte = trap_pte;
166 shadow_notrap_nonpresent_pte = notrap_pte;
167 }
168 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
169
170 static int is_write_protection(struct kvm_vcpu *vcpu)
171 {
172 return vcpu->cr0 & X86_CR0_WP;
173 }
174
175 static int is_cpuid_PSE36(void)
176 {
177 return 1;
178 }
179
180 static int is_nx(struct kvm_vcpu *vcpu)
181 {
182 return vcpu->shadow_efer & EFER_NX;
183 }
184
185 static int is_present_pte(unsigned long pte)
186 {
187 return pte & PT_PRESENT_MASK;
188 }
189
190 static int is_shadow_present_pte(u64 pte)
191 {
192 pte &= ~PT_SHADOW_IO_MARK;
193 return pte != shadow_trap_nonpresent_pte
194 && pte != shadow_notrap_nonpresent_pte;
195 }
196
197 static int is_writeble_pte(unsigned long pte)
198 {
199 return pte & PT_WRITABLE_MASK;
200 }
201
202 static int is_dirty_pte(unsigned long pte)
203 {
204 return pte & PT_DIRTY_MASK;
205 }
206
207 static int is_io_pte(unsigned long pte)
208 {
209 return pte & PT_SHADOW_IO_MARK;
210 }
211
212 static int is_rmap_pte(u64 pte)
213 {
214 return pte != shadow_trap_nonpresent_pte
215 && pte != shadow_notrap_nonpresent_pte;
216 }
217
218 static void set_shadow_pte(u64 *sptep, u64 spte)
219 {
220 #ifdef CONFIG_X86_64
221 set_64bit((unsigned long *)sptep, spte);
222 #else
223 set_64bit((unsigned long long *)sptep, spte);
224 #endif
225 }
226
227 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
228 struct kmem_cache *base_cache, int min)
229 {
230 void *obj;
231
232 if (cache->nobjs >= min)
233 return 0;
234 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
235 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
236 if (!obj)
237 return -ENOMEM;
238 cache->objects[cache->nobjs++] = obj;
239 }
240 return 0;
241 }
242
243 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
244 {
245 while (mc->nobjs)
246 kfree(mc->objects[--mc->nobjs]);
247 }
248
249 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
250 int min)
251 {
252 struct page *page;
253
254 if (cache->nobjs >= min)
255 return 0;
256 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
257 page = alloc_page(GFP_KERNEL);
258 if (!page)
259 return -ENOMEM;
260 set_page_private(page, 0);
261 cache->objects[cache->nobjs++] = page_address(page);
262 }
263 return 0;
264 }
265
266 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
267 {
268 while (mc->nobjs)
269 free_page((unsigned long)mc->objects[--mc->nobjs]);
270 }
271
272 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
273 {
274 int r;
275
276 kvm_mmu_free_some_pages(vcpu);
277 r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
278 pte_chain_cache, 4);
279 if (r)
280 goto out;
281 r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
282 rmap_desc_cache, 1);
283 if (r)
284 goto out;
285 r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 8);
286 if (r)
287 goto out;
288 r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
289 mmu_page_header_cache, 4);
290 out:
291 return r;
292 }
293
294 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
295 {
296 mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
297 mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
298 mmu_free_memory_cache_page(&vcpu->mmu_page_cache);
299 mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
300 }
301
302 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
303 size_t size)
304 {
305 void *p;
306
307 BUG_ON(!mc->nobjs);
308 p = mc->objects[--mc->nobjs];
309 memset(p, 0, size);
310 return p;
311 }
312
313 static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
314 {
315 return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
316 sizeof(struct kvm_pte_chain));
317 }
318
319 static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
320 {
321 kfree(pc);
322 }
323
324 static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
325 {
326 return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
327 sizeof(struct kvm_rmap_desc));
328 }
329
330 static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
331 {
332 kfree(rd);
333 }
334
335 /*
336 * Take gfn and return the reverse mapping to it.
337 * Note: gfn must be unaliased before this function get called
338 */
339
340 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn)
341 {
342 struct kvm_memory_slot *slot;
343
344 slot = gfn_to_memslot(kvm, gfn);
345 return &slot->rmap[gfn - slot->base_gfn];
346 }
347
348 /*
349 * Reverse mapping data structures:
350 *
351 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
352 * that points to page_address(page).
353 *
354 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
355 * containing more mappings.
356 */
357 static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
358 {
359 struct kvm_mmu_page *page;
360 struct kvm_rmap_desc *desc;
361 unsigned long *rmapp;
362 int i;
363
364 if (!is_rmap_pte(*spte))
365 return;
366 gfn = unalias_gfn(vcpu->kvm, gfn);
367 page = page_header(__pa(spte));
368 page->gfns[spte - page->spt] = gfn;
369 rmapp = gfn_to_rmap(vcpu->kvm, gfn);
370 if (!*rmapp) {
371 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
372 *rmapp = (unsigned long)spte;
373 } else if (!(*rmapp & 1)) {
374 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
375 desc = mmu_alloc_rmap_desc(vcpu);
376 desc->shadow_ptes[0] = (u64 *)*rmapp;
377 desc->shadow_ptes[1] = spte;
378 *rmapp = (unsigned long)desc | 1;
379 } else {
380 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
381 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
382 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
383 desc = desc->more;
384 if (desc->shadow_ptes[RMAP_EXT-1]) {
385 desc->more = mmu_alloc_rmap_desc(vcpu);
386 desc = desc->more;
387 }
388 for (i = 0; desc->shadow_ptes[i]; ++i)
389 ;
390 desc->shadow_ptes[i] = spte;
391 }
392 }
393
394 static void rmap_desc_remove_entry(unsigned long *rmapp,
395 struct kvm_rmap_desc *desc,
396 int i,
397 struct kvm_rmap_desc *prev_desc)
398 {
399 int j;
400
401 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
402 ;
403 desc->shadow_ptes[i] = desc->shadow_ptes[j];
404 desc->shadow_ptes[j] = NULL;
405 if (j != 0)
406 return;
407 if (!prev_desc && !desc->more)
408 *rmapp = (unsigned long)desc->shadow_ptes[0];
409 else
410 if (prev_desc)
411 prev_desc->more = desc->more;
412 else
413 *rmapp = (unsigned long)desc->more | 1;
414 mmu_free_rmap_desc(desc);
415 }
416
417 static void rmap_remove(struct kvm *kvm, u64 *spte)
418 {
419 struct kvm_rmap_desc *desc;
420 struct kvm_rmap_desc *prev_desc;
421 struct kvm_mmu_page *page;
422 unsigned long *rmapp;
423 int i;
424
425 if (!is_rmap_pte(*spte))
426 return;
427 page = page_header(__pa(spte));
428 kvm_release_page(pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >>
429 PAGE_SHIFT));
430 rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]);
431 if (!*rmapp) {
432 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
433 BUG();
434 } else if (!(*rmapp & 1)) {
435 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
436 if ((u64 *)*rmapp != spte) {
437 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
438 spte, *spte);
439 BUG();
440 }
441 *rmapp = 0;
442 } else {
443 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
444 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
445 prev_desc = NULL;
446 while (desc) {
447 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
448 if (desc->shadow_ptes[i] == spte) {
449 rmap_desc_remove_entry(rmapp,
450 desc, i,
451 prev_desc);
452 return;
453 }
454 prev_desc = desc;
455 desc = desc->more;
456 }
457 BUG();
458 }
459 }
460
461 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
462 {
463 struct kvm_rmap_desc *desc;
464 struct kvm_rmap_desc *prev_desc;
465 u64 *prev_spte;
466 int i;
467
468 if (!*rmapp)
469 return NULL;
470 else if (!(*rmapp & 1)) {
471 if (!spte)
472 return (u64 *)*rmapp;
473 return NULL;
474 }
475 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
476 prev_desc = NULL;
477 prev_spte = NULL;
478 while (desc) {
479 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
480 if (prev_spte == spte)
481 return desc->shadow_ptes[i];
482 prev_spte = desc->shadow_ptes[i];
483 }
484 desc = desc->more;
485 }
486 return NULL;
487 }
488
489 static void rmap_write_protect(struct kvm *kvm, u64 gfn)
490 {
491 unsigned long *rmapp;
492 u64 *spte;
493
494 gfn = unalias_gfn(kvm, gfn);
495 rmapp = gfn_to_rmap(kvm, gfn);
496
497 spte = rmap_next(kvm, rmapp, NULL);
498 while (spte) {
499 BUG_ON(!spte);
500 BUG_ON(!(*spte & PT_PRESENT_MASK));
501 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
502 if (is_writeble_pte(*spte))
503 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
504 kvm_flush_remote_tlbs(kvm);
505 spte = rmap_next(kvm, rmapp, spte);
506 }
507 }
508
509 #ifdef MMU_DEBUG
510 static int is_empty_shadow_page(u64 *spt)
511 {
512 u64 *pos;
513 u64 *end;
514
515 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
516 if ((*pos & ~PT_SHADOW_IO_MARK) != shadow_trap_nonpresent_pte) {
517 printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
518 pos, *pos);
519 return 0;
520 }
521 return 1;
522 }
523 #endif
524
525 static void kvm_mmu_free_page(struct kvm *kvm,
526 struct kvm_mmu_page *page_head)
527 {
528 ASSERT(is_empty_shadow_page(page_head->spt));
529 list_del(&page_head->link);
530 __free_page(virt_to_page(page_head->spt));
531 __free_page(virt_to_page(page_head->gfns));
532 kfree(page_head);
533 ++kvm->n_free_mmu_pages;
534 }
535
536 static unsigned kvm_page_table_hashfn(gfn_t gfn)
537 {
538 return gfn;
539 }
540
541 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
542 u64 *parent_pte)
543 {
544 struct kvm_mmu_page *page;
545
546 if (!vcpu->kvm->n_free_mmu_pages)
547 return NULL;
548
549 page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
550 sizeof *page);
551 page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
552 page->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
553 set_page_private(virt_to_page(page->spt), (unsigned long)page);
554 list_add(&page->link, &vcpu->kvm->active_mmu_pages);
555 ASSERT(is_empty_shadow_page(page->spt));
556 page->slot_bitmap = 0;
557 page->multimapped = 0;
558 page->parent_pte = parent_pte;
559 --vcpu->kvm->n_free_mmu_pages;
560 return page;
561 }
562
563 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
564 struct kvm_mmu_page *page, u64 *parent_pte)
565 {
566 struct kvm_pte_chain *pte_chain;
567 struct hlist_node *node;
568 int i;
569
570 if (!parent_pte)
571 return;
572 if (!page->multimapped) {
573 u64 *old = page->parent_pte;
574
575 if (!old) {
576 page->parent_pte = parent_pte;
577 return;
578 }
579 page->multimapped = 1;
580 pte_chain = mmu_alloc_pte_chain(vcpu);
581 INIT_HLIST_HEAD(&page->parent_ptes);
582 hlist_add_head(&pte_chain->link, &page->parent_ptes);
583 pte_chain->parent_ptes[0] = old;
584 }
585 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
586 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
587 continue;
588 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
589 if (!pte_chain->parent_ptes[i]) {
590 pte_chain->parent_ptes[i] = parent_pte;
591 return;
592 }
593 }
594 pte_chain = mmu_alloc_pte_chain(vcpu);
595 BUG_ON(!pte_chain);
596 hlist_add_head(&pte_chain->link, &page->parent_ptes);
597 pte_chain->parent_ptes[0] = parent_pte;
598 }
599
600 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page,
601 u64 *parent_pte)
602 {
603 struct kvm_pte_chain *pte_chain;
604 struct hlist_node *node;
605 int i;
606
607 if (!page->multimapped) {
608 BUG_ON(page->parent_pte != parent_pte);
609 page->parent_pte = NULL;
610 return;
611 }
612 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
613 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
614 if (!pte_chain->parent_ptes[i])
615 break;
616 if (pte_chain->parent_ptes[i] != parent_pte)
617 continue;
618 while (i + 1 < NR_PTE_CHAIN_ENTRIES
619 && pte_chain->parent_ptes[i + 1]) {
620 pte_chain->parent_ptes[i]
621 = pte_chain->parent_ptes[i + 1];
622 ++i;
623 }
624 pte_chain->parent_ptes[i] = NULL;
625 if (i == 0) {
626 hlist_del(&pte_chain->link);
627 mmu_free_pte_chain(pte_chain);
628 if (hlist_empty(&page->parent_ptes)) {
629 page->multimapped = 0;
630 page->parent_pte = NULL;
631 }
632 }
633 return;
634 }
635 BUG();
636 }
637
638 static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm,
639 gfn_t gfn)
640 {
641 unsigned index;
642 struct hlist_head *bucket;
643 struct kvm_mmu_page *page;
644 struct hlist_node *node;
645
646 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
647 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
648 bucket = &kvm->mmu_page_hash[index];
649 hlist_for_each_entry(page, node, bucket, hash_link)
650 if (page->gfn == gfn && !page->role.metaphysical) {
651 pgprintk("%s: found role %x\n",
652 __FUNCTION__, page->role.word);
653 return page;
654 }
655 return NULL;
656 }
657
658 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
659 gfn_t gfn,
660 gva_t gaddr,
661 unsigned level,
662 int metaphysical,
663 unsigned hugepage_access,
664 u64 *parent_pte)
665 {
666 union kvm_mmu_page_role role;
667 unsigned index;
668 unsigned quadrant;
669 struct hlist_head *bucket;
670 struct kvm_mmu_page *page;
671 struct hlist_node *node;
672
673 role.word = 0;
674 role.glevels = vcpu->mmu.root_level;
675 role.level = level;
676 role.metaphysical = metaphysical;
677 role.hugepage_access = hugepage_access;
678 if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
679 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
680 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
681 role.quadrant = quadrant;
682 }
683 pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
684 gfn, role.word);
685 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
686 bucket = &vcpu->kvm->mmu_page_hash[index];
687 hlist_for_each_entry(page, node, bucket, hash_link)
688 if (page->gfn == gfn && page->role.word == role.word) {
689 mmu_page_add_parent_pte(vcpu, page, parent_pte);
690 pgprintk("%s: found\n", __FUNCTION__);
691 return page;
692 }
693 page = kvm_mmu_alloc_page(vcpu, parent_pte);
694 if (!page)
695 return page;
696 pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
697 page->gfn = gfn;
698 page->role = role;
699 hlist_add_head(&page->hash_link, bucket);
700 vcpu->mmu.prefetch_page(vcpu, page);
701 if (!metaphysical)
702 rmap_write_protect(vcpu->kvm, gfn);
703 return page;
704 }
705
706 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
707 struct kvm_mmu_page *page)
708 {
709 unsigned i;
710 u64 *pt;
711 u64 ent;
712
713 pt = page->spt;
714
715 if (page->role.level == PT_PAGE_TABLE_LEVEL) {
716 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
717 if (is_shadow_present_pte(pt[i]))
718 rmap_remove(kvm, &pt[i]);
719 pt[i] = shadow_trap_nonpresent_pte;
720 }
721 kvm_flush_remote_tlbs(kvm);
722 return;
723 }
724
725 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
726 ent = pt[i];
727
728 pt[i] = shadow_trap_nonpresent_pte;
729 if (!is_shadow_present_pte(ent))
730 continue;
731 ent &= PT64_BASE_ADDR_MASK;
732 mmu_page_remove_parent_pte(page_header(ent), &pt[i]);
733 }
734 kvm_flush_remote_tlbs(kvm);
735 }
736
737 static void kvm_mmu_put_page(struct kvm_mmu_page *page,
738 u64 *parent_pte)
739 {
740 mmu_page_remove_parent_pte(page, parent_pte);
741 }
742
743 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
744 {
745 int i;
746
747 for (i = 0; i < KVM_MAX_VCPUS; ++i)
748 if (kvm->vcpus[i])
749 kvm->vcpus[i]->last_pte_updated = NULL;
750 }
751
752 static void kvm_mmu_zap_page(struct kvm *kvm,
753 struct kvm_mmu_page *page)
754 {
755 u64 *parent_pte;
756
757 while (page->multimapped || page->parent_pte) {
758 if (!page->multimapped)
759 parent_pte = page->parent_pte;
760 else {
761 struct kvm_pte_chain *chain;
762
763 chain = container_of(page->parent_ptes.first,
764 struct kvm_pte_chain, link);
765 parent_pte = chain->parent_ptes[0];
766 }
767 BUG_ON(!parent_pte);
768 kvm_mmu_put_page(page, parent_pte);
769 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
770 }
771 kvm_mmu_page_unlink_children(kvm, page);
772 if (!page->root_count) {
773 hlist_del(&page->hash_link);
774 kvm_mmu_free_page(kvm, page);
775 } else
776 list_move(&page->link, &kvm->active_mmu_pages);
777 kvm_mmu_reset_last_pte_updated(kvm);
778 }
779
780 /*
781 * Changing the number of mmu pages allocated to the vm
782 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
783 */
784 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
785 {
786 /*
787 * If we set the number of mmu pages to be smaller be than the
788 * number of actived pages , we must to free some mmu pages before we
789 * change the value
790 */
791
792 if ((kvm->n_alloc_mmu_pages - kvm->n_free_mmu_pages) >
793 kvm_nr_mmu_pages) {
794 int n_used_mmu_pages = kvm->n_alloc_mmu_pages
795 - kvm->n_free_mmu_pages;
796
797 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
798 struct kvm_mmu_page *page;
799
800 page = container_of(kvm->active_mmu_pages.prev,
801 struct kvm_mmu_page, link);
802 kvm_mmu_zap_page(kvm, page);
803 n_used_mmu_pages--;
804 }
805 kvm->n_free_mmu_pages = 0;
806 }
807 else
808 kvm->n_free_mmu_pages += kvm_nr_mmu_pages
809 - kvm->n_alloc_mmu_pages;
810
811 kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages;
812 }
813
814 static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
815 {
816 unsigned index;
817 struct hlist_head *bucket;
818 struct kvm_mmu_page *page;
819 struct hlist_node *node, *n;
820 int r;
821
822 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
823 r = 0;
824 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
825 bucket = &kvm->mmu_page_hash[index];
826 hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
827 if (page->gfn == gfn && !page->role.metaphysical) {
828 pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
829 page->role.word);
830 kvm_mmu_zap_page(kvm, page);
831 r = 1;
832 }
833 return r;
834 }
835
836 static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
837 {
838 struct kvm_mmu_page *page;
839
840 while ((page = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
841 pgprintk("%s: zap %lx %x\n",
842 __FUNCTION__, gfn, page->role.word);
843 kvm_mmu_zap_page(kvm, page);
844 }
845 }
846
847 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
848 {
849 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
850 struct kvm_mmu_page *page_head = page_header(__pa(pte));
851
852 __set_bit(slot, &page_head->slot_bitmap);
853 }
854
855 hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa)
856 {
857 struct page *page;
858 hpa_t hpa;
859
860 ASSERT((gpa & HPA_ERR_MASK) == 0);
861 page = gfn_to_page(kvm, gpa >> PAGE_SHIFT);
862 hpa = ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) | (gpa & (PAGE_SIZE-1));
863 if (is_error_page(page))
864 return hpa | HPA_ERR_MASK;
865 return hpa;
866 }
867
868 hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
869 {
870 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
871
872 if (gpa == UNMAPPED_GVA)
873 return UNMAPPED_GVA;
874 return gpa_to_hpa(vcpu->kvm, gpa);
875 }
876
877 struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
878 {
879 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
880
881 if (gpa == UNMAPPED_GVA)
882 return NULL;
883 return pfn_to_page(gpa_to_hpa(vcpu->kvm, gpa) >> PAGE_SHIFT);
884 }
885
886 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
887 {
888 }
889
890 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
891 {
892 int level = PT32E_ROOT_LEVEL;
893 hpa_t table_addr = vcpu->mmu.root_hpa;
894
895 for (; ; level--) {
896 u32 index = PT64_INDEX(v, level);
897 u64 *table;
898 u64 pte;
899
900 ASSERT(VALID_PAGE(table_addr));
901 table = __va(table_addr);
902
903 if (level == 1) {
904 int was_rmapped;
905
906 pte = table[index];
907 was_rmapped = is_rmap_pte(pte);
908 if (is_shadow_present_pte(pte) && is_writeble_pte(pte))
909 return 0;
910 mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
911 page_header_update_slot(vcpu->kvm, table, v);
912 table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
913 PT_USER_MASK;
914 if (!was_rmapped)
915 rmap_add(vcpu, &table[index], v >> PAGE_SHIFT);
916 else
917 kvm_release_page(pfn_to_page(p >> PAGE_SHIFT));
918 return 0;
919 }
920
921 if (table[index] == shadow_trap_nonpresent_pte) {
922 struct kvm_mmu_page *new_table;
923 gfn_t pseudo_gfn;
924
925 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
926 >> PAGE_SHIFT;
927 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
928 v, level - 1,
929 1, 3, &table[index]);
930 if (!new_table) {
931 pgprintk("nonpaging_map: ENOMEM\n");
932 kvm_release_page(pfn_to_page(p >> PAGE_SHIFT));
933 return -ENOMEM;
934 }
935
936 table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
937 | PT_WRITABLE_MASK | PT_USER_MASK;
938 }
939 table_addr = table[index] & PT64_BASE_ADDR_MASK;
940 }
941 }
942
943 static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
944 struct kvm_mmu_page *sp)
945 {
946 int i;
947
948 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
949 sp->spt[i] = shadow_trap_nonpresent_pte;
950 }
951
952 static void mmu_free_roots(struct kvm_vcpu *vcpu)
953 {
954 int i;
955 struct kvm_mmu_page *page;
956
957 if (!VALID_PAGE(vcpu->mmu.root_hpa))
958 return;
959 #ifdef CONFIG_X86_64
960 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
961 hpa_t root = vcpu->mmu.root_hpa;
962
963 page = page_header(root);
964 --page->root_count;
965 vcpu->mmu.root_hpa = INVALID_PAGE;
966 return;
967 }
968 #endif
969 for (i = 0; i < 4; ++i) {
970 hpa_t root = vcpu->mmu.pae_root[i];
971
972 if (root) {
973 root &= PT64_BASE_ADDR_MASK;
974 page = page_header(root);
975 --page->root_count;
976 }
977 vcpu->mmu.pae_root[i] = INVALID_PAGE;
978 }
979 vcpu->mmu.root_hpa = INVALID_PAGE;
980 }
981
982 static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
983 {
984 int i;
985 gfn_t root_gfn;
986 struct kvm_mmu_page *page;
987
988 root_gfn = vcpu->cr3 >> PAGE_SHIFT;
989
990 #ifdef CONFIG_X86_64
991 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
992 hpa_t root = vcpu->mmu.root_hpa;
993
994 ASSERT(!VALID_PAGE(root));
995 page = kvm_mmu_get_page(vcpu, root_gfn, 0,
996 PT64_ROOT_LEVEL, 0, 0, NULL);
997 root = __pa(page->spt);
998 ++page->root_count;
999 vcpu->mmu.root_hpa = root;
1000 return;
1001 }
1002 #endif
1003 for (i = 0; i < 4; ++i) {
1004 hpa_t root = vcpu->mmu.pae_root[i];
1005
1006 ASSERT(!VALID_PAGE(root));
1007 if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
1008 if (!is_present_pte(vcpu->pdptrs[i])) {
1009 vcpu->mmu.pae_root[i] = 0;
1010 continue;
1011 }
1012 root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
1013 } else if (vcpu->mmu.root_level == 0)
1014 root_gfn = 0;
1015 page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
1016 PT32_ROOT_LEVEL, !is_paging(vcpu),
1017 0, NULL);
1018 root = __pa(page->spt);
1019 ++page->root_count;
1020 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
1021 }
1022 vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
1023 }
1024
1025 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
1026 {
1027 return vaddr;
1028 }
1029
1030 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
1031 u32 error_code)
1032 {
1033 gpa_t addr = gva;
1034 hpa_t paddr;
1035 int r;
1036
1037 r = mmu_topup_memory_caches(vcpu);
1038 if (r)
1039 return r;
1040
1041 ASSERT(vcpu);
1042 ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
1043
1044
1045 paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK);
1046
1047 if (is_error_hpa(paddr)) {
1048 kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK)
1049 >> PAGE_SHIFT));
1050 return 1;
1051 }
1052
1053 return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
1054 }
1055
1056 static void nonpaging_free(struct kvm_vcpu *vcpu)
1057 {
1058 mmu_free_roots(vcpu);
1059 }
1060
1061 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
1062 {
1063 struct kvm_mmu *context = &vcpu->mmu;
1064
1065 context->new_cr3 = nonpaging_new_cr3;
1066 context->page_fault = nonpaging_page_fault;
1067 context->gva_to_gpa = nonpaging_gva_to_gpa;
1068 context->free = nonpaging_free;
1069 context->prefetch_page = nonpaging_prefetch_page;
1070 context->root_level = 0;
1071 context->shadow_root_level = PT32E_ROOT_LEVEL;
1072 context->root_hpa = INVALID_PAGE;
1073 return 0;
1074 }
1075
1076 static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
1077 {
1078 ++vcpu->stat.tlb_flush;
1079 kvm_x86_ops->tlb_flush(vcpu);
1080 }
1081
1082 static void paging_new_cr3(struct kvm_vcpu *vcpu)
1083 {
1084 pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
1085 mmu_free_roots(vcpu);
1086 }
1087
1088 static void inject_page_fault(struct kvm_vcpu *vcpu,
1089 u64 addr,
1090 u32 err_code)
1091 {
1092 kvm_x86_ops->inject_page_fault(vcpu, addr, err_code);
1093 }
1094
1095 static void paging_free(struct kvm_vcpu *vcpu)
1096 {
1097 nonpaging_free(vcpu);
1098 }
1099
1100 #define PTTYPE 64
1101 #include "paging_tmpl.h"
1102 #undef PTTYPE
1103
1104 #define PTTYPE 32
1105 #include "paging_tmpl.h"
1106 #undef PTTYPE
1107
1108 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
1109 {
1110 struct kvm_mmu *context = &vcpu->mmu;
1111
1112 ASSERT(is_pae(vcpu));
1113 context->new_cr3 = paging_new_cr3;
1114 context->page_fault = paging64_page_fault;
1115 context->gva_to_gpa = paging64_gva_to_gpa;
1116 context->prefetch_page = paging64_prefetch_page;
1117 context->free = paging_free;
1118 context->root_level = level;
1119 context->shadow_root_level = level;
1120 context->root_hpa = INVALID_PAGE;
1121 return 0;
1122 }
1123
1124 static int paging64_init_context(struct kvm_vcpu *vcpu)
1125 {
1126 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1127 }
1128
1129 static int paging32_init_context(struct kvm_vcpu *vcpu)
1130 {
1131 struct kvm_mmu *context = &vcpu->mmu;
1132
1133 context->new_cr3 = paging_new_cr3;
1134 context->page_fault = paging32_page_fault;
1135 context->gva_to_gpa = paging32_gva_to_gpa;
1136 context->free = paging_free;
1137 context->prefetch_page = paging32_prefetch_page;
1138 context->root_level = PT32_ROOT_LEVEL;
1139 context->shadow_root_level = PT32E_ROOT_LEVEL;
1140 context->root_hpa = INVALID_PAGE;
1141 return 0;
1142 }
1143
1144 static int paging32E_init_context(struct kvm_vcpu *vcpu)
1145 {
1146 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
1147 }
1148
1149 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1150 {
1151 ASSERT(vcpu);
1152 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1153
1154 if (!is_paging(vcpu))
1155 return nonpaging_init_context(vcpu);
1156 else if (is_long_mode(vcpu))
1157 return paging64_init_context(vcpu);
1158 else if (is_pae(vcpu))
1159 return paging32E_init_context(vcpu);
1160 else
1161 return paging32_init_context(vcpu);
1162 }
1163
1164 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1165 {
1166 ASSERT(vcpu);
1167 if (VALID_PAGE(vcpu->mmu.root_hpa)) {
1168 vcpu->mmu.free(vcpu);
1169 vcpu->mmu.root_hpa = INVALID_PAGE;
1170 }
1171 }
1172
1173 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1174 {
1175 destroy_kvm_mmu(vcpu);
1176 return init_kvm_mmu(vcpu);
1177 }
1178 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
1179
1180 int kvm_mmu_load(struct kvm_vcpu *vcpu)
1181 {
1182 int r;
1183
1184 mutex_lock(&vcpu->kvm->lock);
1185 r = mmu_topup_memory_caches(vcpu);
1186 if (r)
1187 goto out;
1188 mmu_alloc_roots(vcpu);
1189 kvm_x86_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
1190 kvm_mmu_flush_tlb(vcpu);
1191 out:
1192 mutex_unlock(&vcpu->kvm->lock);
1193 return r;
1194 }
1195 EXPORT_SYMBOL_GPL(kvm_mmu_load);
1196
1197 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
1198 {
1199 mmu_free_roots(vcpu);
1200 }
1201
1202 static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
1203 struct kvm_mmu_page *page,
1204 u64 *spte)
1205 {
1206 u64 pte;
1207 struct kvm_mmu_page *child;
1208
1209 pte = *spte;
1210 if (is_shadow_present_pte(pte)) {
1211 if (page->role.level == PT_PAGE_TABLE_LEVEL)
1212 rmap_remove(vcpu->kvm, spte);
1213 else {
1214 child = page_header(pte & PT64_BASE_ADDR_MASK);
1215 mmu_page_remove_parent_pte(child, spte);
1216 }
1217 }
1218 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
1219 kvm_flush_remote_tlbs(vcpu->kvm);
1220 }
1221
1222 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
1223 struct kvm_mmu_page *page,
1224 u64 *spte,
1225 const void *new, int bytes,
1226 int offset_in_pte)
1227 {
1228 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1229 return;
1230
1231 if (page->role.glevels == PT32_ROOT_LEVEL)
1232 paging32_update_pte(vcpu, page, spte, new, bytes,
1233 offset_in_pte);
1234 else
1235 paging64_update_pte(vcpu, page, spte, new, bytes,
1236 offset_in_pte);
1237 }
1238
1239 static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
1240 {
1241 u64 *spte = vcpu->last_pte_updated;
1242
1243 return !!(spte && (*spte & PT_ACCESSED_MASK));
1244 }
1245
1246 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1247 const u8 *new, int bytes)
1248 {
1249 gfn_t gfn = gpa >> PAGE_SHIFT;
1250 struct kvm_mmu_page *page;
1251 struct hlist_node *node, *n;
1252 struct hlist_head *bucket;
1253 unsigned index;
1254 u64 *spte;
1255 unsigned offset = offset_in_page(gpa);
1256 unsigned pte_size;
1257 unsigned page_offset;
1258 unsigned misaligned;
1259 unsigned quadrant;
1260 int level;
1261 int flooded = 0;
1262 int npte;
1263
1264 pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
1265 kvm_mmu_audit(vcpu, "pre pte write");
1266 if (gfn == vcpu->last_pt_write_gfn
1267 && !last_updated_pte_accessed(vcpu)) {
1268 ++vcpu->last_pt_write_count;
1269 if (vcpu->last_pt_write_count >= 3)
1270 flooded = 1;
1271 } else {
1272 vcpu->last_pt_write_gfn = gfn;
1273 vcpu->last_pt_write_count = 1;
1274 vcpu->last_pte_updated = NULL;
1275 }
1276 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
1277 bucket = &vcpu->kvm->mmu_page_hash[index];
1278 hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
1279 if (page->gfn != gfn || page->role.metaphysical)
1280 continue;
1281 pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1282 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
1283 misaligned |= bytes < 4;
1284 if (misaligned || flooded) {
1285 /*
1286 * Misaligned accesses are too much trouble to fix
1287 * up; also, they usually indicate a page is not used
1288 * as a page table.
1289 *
1290 * If we're seeing too many writes to a page,
1291 * it may no longer be a page table, or we may be
1292 * forking, in which case it is better to unmap the
1293 * page.
1294 */
1295 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1296 gpa, bytes, page->role.word);
1297 kvm_mmu_zap_page(vcpu->kvm, page);
1298 continue;
1299 }
1300 page_offset = offset;
1301 level = page->role.level;
1302 npte = 1;
1303 if (page->role.glevels == PT32_ROOT_LEVEL) {
1304 page_offset <<= 1; /* 32->64 */
1305 /*
1306 * A 32-bit pde maps 4MB while the shadow pdes map
1307 * only 2MB. So we need to double the offset again
1308 * and zap two pdes instead of one.
1309 */
1310 if (level == PT32_ROOT_LEVEL) {
1311 page_offset &= ~7; /* kill rounding error */
1312 page_offset <<= 1;
1313 npte = 2;
1314 }
1315 quadrant = page_offset >> PAGE_SHIFT;
1316 page_offset &= ~PAGE_MASK;
1317 if (quadrant != page->role.quadrant)
1318 continue;
1319 }
1320 spte = &page->spt[page_offset / sizeof(*spte)];
1321 while (npte--) {
1322 mmu_pte_write_zap_pte(vcpu, page, spte);
1323 mmu_pte_write_new_pte(vcpu, page, spte, new, bytes,
1324 page_offset & (pte_size - 1));
1325 ++spte;
1326 }
1327 }
1328 kvm_mmu_audit(vcpu, "post pte write");
1329 }
1330
1331 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1332 {
1333 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
1334
1335 return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1336 }
1337
1338 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
1339 {
1340 while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
1341 struct kvm_mmu_page *page;
1342
1343 page = container_of(vcpu->kvm->active_mmu_pages.prev,
1344 struct kvm_mmu_page, link);
1345 kvm_mmu_zap_page(vcpu->kvm, page);
1346 }
1347 }
1348
1349 static void free_mmu_pages(struct kvm_vcpu *vcpu)
1350 {
1351 struct kvm_mmu_page *page;
1352
1353 while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
1354 page = container_of(vcpu->kvm->active_mmu_pages.next,
1355 struct kvm_mmu_page, link);
1356 kvm_mmu_zap_page(vcpu->kvm, page);
1357 }
1358 free_page((unsigned long)vcpu->mmu.pae_root);
1359 }
1360
1361 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1362 {
1363 struct page *page;
1364 int i;
1365
1366 ASSERT(vcpu);
1367
1368 if (vcpu->kvm->n_requested_mmu_pages)
1369 vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_requested_mmu_pages;
1370 else
1371 vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_alloc_mmu_pages;
1372 /*
1373 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1374 * Therefore we need to allocate shadow page tables in the first
1375 * 4GB of memory, which happens to fit the DMA32 zone.
1376 */
1377 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1378 if (!page)
1379 goto error_1;
1380 vcpu->mmu.pae_root = page_address(page);
1381 for (i = 0; i < 4; ++i)
1382 vcpu->mmu.pae_root[i] = INVALID_PAGE;
1383
1384 return 0;
1385
1386 error_1:
1387 free_mmu_pages(vcpu);
1388 return -ENOMEM;
1389 }
1390
1391 int kvm_mmu_create(struct kvm_vcpu *vcpu)
1392 {
1393 ASSERT(vcpu);
1394 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1395
1396 return alloc_mmu_pages(vcpu);
1397 }
1398
1399 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1400 {
1401 ASSERT(vcpu);
1402 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1403
1404 return init_kvm_mmu(vcpu);
1405 }
1406
1407 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1408 {
1409 ASSERT(vcpu);
1410
1411 destroy_kvm_mmu(vcpu);
1412 free_mmu_pages(vcpu);
1413 mmu_free_memory_caches(vcpu);
1414 }
1415
1416 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
1417 {
1418 struct kvm_mmu_page *page;
1419
1420 list_for_each_entry(page, &kvm->active_mmu_pages, link) {
1421 int i;
1422 u64 *pt;
1423
1424 if (!test_bit(slot, &page->slot_bitmap))
1425 continue;
1426
1427 pt = page->spt;
1428 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1429 /* avoid RMW */
1430 if (pt[i] & PT_WRITABLE_MASK)
1431 pt[i] &= ~PT_WRITABLE_MASK;
1432 }
1433 }
1434
1435 void kvm_mmu_zap_all(struct kvm *kvm)
1436 {
1437 struct kvm_mmu_page *page, *node;
1438
1439 list_for_each_entry_safe(page, node, &kvm->active_mmu_pages, link)
1440 kvm_mmu_zap_page(kvm, page);
1441
1442 kvm_flush_remote_tlbs(kvm);
1443 }
1444
1445 void kvm_mmu_module_exit(void)
1446 {
1447 if (pte_chain_cache)
1448 kmem_cache_destroy(pte_chain_cache);
1449 if (rmap_desc_cache)
1450 kmem_cache_destroy(rmap_desc_cache);
1451 if (mmu_page_header_cache)
1452 kmem_cache_destroy(mmu_page_header_cache);
1453 }
1454
1455 int kvm_mmu_module_init(void)
1456 {
1457 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
1458 sizeof(struct kvm_pte_chain),
1459 0, 0, NULL);
1460 if (!pte_chain_cache)
1461 goto nomem;
1462 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
1463 sizeof(struct kvm_rmap_desc),
1464 0, 0, NULL);
1465 if (!rmap_desc_cache)
1466 goto nomem;
1467
1468 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
1469 sizeof(struct kvm_mmu_page),
1470 0, 0, NULL);
1471 if (!mmu_page_header_cache)
1472 goto nomem;
1473
1474 return 0;
1475
1476 nomem:
1477 kvm_mmu_module_exit();
1478 return -ENOMEM;
1479 }
1480
1481 #ifdef AUDIT
1482
1483 static const char *audit_msg;
1484
1485 static gva_t canonicalize(gva_t gva)
1486 {
1487 #ifdef CONFIG_X86_64
1488 gva = (long long)(gva << 16) >> 16;
1489 #endif
1490 return gva;
1491 }
1492
1493 static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
1494 gva_t va, int level)
1495 {
1496 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
1497 int i;
1498 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
1499
1500 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
1501 u64 ent = pt[i];
1502
1503 if (ent == shadow_trap_nonpresent_pte)
1504 continue;
1505
1506 va = canonicalize(va);
1507 if (level > 1) {
1508 if (ent == shadow_notrap_nonpresent_pte)
1509 printk(KERN_ERR "audit: (%s) nontrapping pte"
1510 " in nonleaf level: levels %d gva %lx"
1511 " level %d pte %llx\n", audit_msg,
1512 vcpu->mmu.root_level, va, level, ent);
1513
1514 audit_mappings_page(vcpu, ent, va, level - 1);
1515 } else {
1516 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
1517 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
1518 struct page *page;
1519
1520 if (is_shadow_present_pte(ent)
1521 && (ent & PT64_BASE_ADDR_MASK) != hpa)
1522 printk(KERN_ERR "xx audit error: (%s) levels %d"
1523 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
1524 audit_msg, vcpu->mmu.root_level,
1525 va, gpa, hpa, ent,
1526 is_shadow_present_pte(ent));
1527 else if (ent == shadow_notrap_nonpresent_pte
1528 && !is_error_hpa(hpa))
1529 printk(KERN_ERR "audit: (%s) notrap shadow,"
1530 " valid guest gva %lx\n", audit_msg, va);
1531 page = pfn_to_page((gpa & PT64_BASE_ADDR_MASK)
1532 >> PAGE_SHIFT);
1533 kvm_release_page(page);
1534
1535 }
1536 }
1537 }
1538
1539 static void audit_mappings(struct kvm_vcpu *vcpu)
1540 {
1541 unsigned i;
1542
1543 if (vcpu->mmu.root_level == 4)
1544 audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
1545 else
1546 for (i = 0; i < 4; ++i)
1547 if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
1548 audit_mappings_page(vcpu,
1549 vcpu->mmu.pae_root[i],
1550 i << 30,
1551 2);
1552 }
1553
1554 static int count_rmaps(struct kvm_vcpu *vcpu)
1555 {
1556 int nmaps = 0;
1557 int i, j, k;
1558
1559 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
1560 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
1561 struct kvm_rmap_desc *d;
1562
1563 for (j = 0; j < m->npages; ++j) {
1564 unsigned long *rmapp = &m->rmap[j];
1565
1566 if (!*rmapp)
1567 continue;
1568 if (!(*rmapp & 1)) {
1569 ++nmaps;
1570 continue;
1571 }
1572 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
1573 while (d) {
1574 for (k = 0; k < RMAP_EXT; ++k)
1575 if (d->shadow_ptes[k])
1576 ++nmaps;
1577 else
1578 break;
1579 d = d->more;
1580 }
1581 }
1582 }
1583 return nmaps;
1584 }
1585
1586 static int count_writable_mappings(struct kvm_vcpu *vcpu)
1587 {
1588 int nmaps = 0;
1589 struct kvm_mmu_page *page;
1590 int i;
1591
1592 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1593 u64 *pt = page->spt;
1594
1595 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1596 continue;
1597
1598 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1599 u64 ent = pt[i];
1600
1601 if (!(ent & PT_PRESENT_MASK))
1602 continue;
1603 if (!(ent & PT_WRITABLE_MASK))
1604 continue;
1605 ++nmaps;
1606 }
1607 }
1608 return nmaps;
1609 }
1610
1611 static void audit_rmap(struct kvm_vcpu *vcpu)
1612 {
1613 int n_rmap = count_rmaps(vcpu);
1614 int n_actual = count_writable_mappings(vcpu);
1615
1616 if (n_rmap != n_actual)
1617 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
1618 __FUNCTION__, audit_msg, n_rmap, n_actual);
1619 }
1620
1621 static void audit_write_protection(struct kvm_vcpu *vcpu)
1622 {
1623 struct kvm_mmu_page *page;
1624 struct kvm_memory_slot *slot;
1625 unsigned long *rmapp;
1626 gfn_t gfn;
1627
1628 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1629 if (page->role.metaphysical)
1630 continue;
1631
1632 slot = gfn_to_memslot(vcpu->kvm, page->gfn);
1633 gfn = unalias_gfn(vcpu->kvm, page->gfn);
1634 rmapp = &slot->rmap[gfn - slot->base_gfn];
1635 if (*rmapp)
1636 printk(KERN_ERR "%s: (%s) shadow page has writable"
1637 " mappings: gfn %lx role %x\n",
1638 __FUNCTION__, audit_msg, page->gfn,
1639 page->role.word);
1640 }
1641 }
1642
1643 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
1644 {
1645 int olddbg = dbg;
1646
1647 dbg = 0;
1648 audit_msg = msg;
1649 audit_rmap(vcpu);
1650 audit_write_protection(vcpu);
1651 audit_mappings(vcpu);
1652 dbg = olddbg;
1653 }
1654
1655 #endif
Linux kernel - Deliverables
This page took 0.065128 seconds and 5 git commands to generate.