projects / deliverable / linux.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
KVM: MMU: fix broken page accessed tracking with ept enabled
[deliverable/linux.git] / arch / x86 / kvm / mmu.c
... / ...
CommitLineData
1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 * Copyright 2010 Red Hat, Inc. and/or its affilates.
11 *
12 * Authors:
13 * Yaniv Kamay <yaniv@qumranet.com>
14 * Avi Kivity <avi@qumranet.com>
15 *
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
18 *
19 */
20
21#include "mmu.h"
22#include "x86.h"
23#include "kvm_cache_regs.h"
24
25#include <linux/kvm_host.h>
26#include <linux/types.h>
27#include <linux/string.h>
28#include <linux/mm.h>
29#include <linux/highmem.h>
30#include <linux/module.h>
31#include <linux/swap.h>
32#include <linux/hugetlb.h>
33#include <linux/compiler.h>
34#include <linux/srcu.h>
35#include <linux/slab.h>
36#include <linux/uaccess.h>
37
38#include <asm/page.h>
39#include <asm/cmpxchg.h>
40#include <asm/io.h>
41#include <asm/vmx.h>
42
43/*
44 * When setting this variable to true it enables Two-Dimensional-Paging
45 * where the hardware walks 2 page tables:
46 * 1. the guest-virtual to guest-physical
47 * 2. while doing 1. it walks guest-physical to host-physical
48 * If the hardware supports that we don't need to do shadow paging.
49 */
50bool tdp_enabled = false;
51
52#undef MMU_DEBUG
53
54#undef AUDIT
55
56#ifdef AUDIT
57static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
58#else
59static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
60#endif
61
62#ifdef MMU_DEBUG
63
64#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
65#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
66
67#else
68
69#define pgprintk(x...) do { } while (0)
70#define rmap_printk(x...) do { } while (0)
71
72#endif
73
74#if defined(MMU_DEBUG) || defined(AUDIT)
75static int dbg = 0;
76module_param(dbg, bool, 0644);
77#endif
78
79static int oos_shadow = 1;
80module_param(oos_shadow, bool, 0644);
81
82#ifndef MMU_DEBUG
83#define ASSERT(x) do { } while (0)
84#else
85#define ASSERT(x) \
86 if (!(x)) { \
87 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
88 __FILE__, __LINE__, #x); \
89 }
90#endif
91
92#define PT_FIRST_AVAIL_BITS_SHIFT 9
93#define PT64_SECOND_AVAIL_BITS_SHIFT 52
94
95#define PT64_LEVEL_BITS 9
96
97#define PT64_LEVEL_SHIFT(level) \
98 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
99
100#define PT64_LEVEL_MASK(level) \
101 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
102
103#define PT64_INDEX(address, level)\
104 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
105
106
107#define PT32_LEVEL_BITS 10
108
109#define PT32_LEVEL_SHIFT(level) \
110 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
111
112#define PT32_LEVEL_MASK(level) \
113 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
114#define PT32_LVL_OFFSET_MASK(level) \
115 (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
116 * PT32_LEVEL_BITS))) - 1))
117
118#define PT32_INDEX(address, level)\
119 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
120
121
122#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
123#define PT64_DIR_BASE_ADDR_MASK \
124 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
125#define PT64_LVL_ADDR_MASK(level) \
126 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
127 * PT64_LEVEL_BITS))) - 1))
128#define PT64_LVL_OFFSET_MASK(level) \
129 (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
130 * PT64_LEVEL_BITS))) - 1))
131
132#define PT32_BASE_ADDR_MASK PAGE_MASK
133#define PT32_DIR_BASE_ADDR_MASK \
134 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
135#define PT32_LVL_ADDR_MASK(level) \
136 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
137 * PT32_LEVEL_BITS))) - 1))
138
139#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
140 | PT64_NX_MASK)
141
142#define RMAP_EXT 4
143
144#define ACC_EXEC_MASK 1
145#define ACC_WRITE_MASK PT_WRITABLE_MASK
146#define ACC_USER_MASK PT_USER_MASK
147#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
148
149#include <trace/events/kvm.h>
150
151#define CREATE_TRACE_POINTS
152#include "mmutrace.h"
153
154#define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
155
156#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
157
158struct kvm_rmap_desc {
159 u64 *sptes[RMAP_EXT];
160 struct kvm_rmap_desc *more;
161};
162
163struct kvm_shadow_walk_iterator {
164 u64 addr;
165 hpa_t shadow_addr;
166 int level;
167 u64 *sptep;
168 unsigned index;
169};
170
171#define for_each_shadow_entry(_vcpu, _addr, _walker) \
172 for (shadow_walk_init(&(_walker), _vcpu, _addr); \
173 shadow_walk_okay(&(_walker)); \
174 shadow_walk_next(&(_walker)))
175
176typedef void (*mmu_parent_walk_fn) (struct kvm_mmu_page *sp, u64 *spte);
177
178static struct kmem_cache *pte_chain_cache;
179static struct kmem_cache *rmap_desc_cache;
180static struct kmem_cache *mmu_page_header_cache;
181
182static u64 __read_mostly shadow_trap_nonpresent_pte;
183static u64 __read_mostly shadow_notrap_nonpresent_pte;
184static u64 __read_mostly shadow_base_present_pte;
185static u64 __read_mostly shadow_nx_mask;
186static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
187static u64 __read_mostly shadow_user_mask;
188static u64 __read_mostly shadow_accessed_mask;
189static u64 __read_mostly shadow_dirty_mask;
190
191static inline u64 rsvd_bits(int s, int e)
192{
193 return ((1ULL << (e - s + 1)) - 1) << s;
194}
195
196void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
197{
198 shadow_trap_nonpresent_pte = trap_pte;
199 shadow_notrap_nonpresent_pte = notrap_pte;
200}
201EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
202
203void kvm_mmu_set_base_ptes(u64 base_pte)
204{
205 shadow_base_present_pte = base_pte;
206}
207EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
208
209void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
210 u64 dirty_mask, u64 nx_mask, u64 x_mask)
211{
212 shadow_user_mask = user_mask;
213 shadow_accessed_mask = accessed_mask;
214 shadow_dirty_mask = dirty_mask;
215 shadow_nx_mask = nx_mask;
216 shadow_x_mask = x_mask;
217}
218EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
219
220static bool is_write_protection(struct kvm_vcpu *vcpu)
221{
222 return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
223}
224
225static int is_cpuid_PSE36(void)
226{
227 return 1;
228}
229
230static int is_nx(struct kvm_vcpu *vcpu)
231{
232 return vcpu->arch.efer & EFER_NX;
233}
234
235static int is_shadow_present_pte(u64 pte)
236{
237 return pte != shadow_trap_nonpresent_pte
238 && pte != shadow_notrap_nonpresent_pte;
239}
240
241static int is_large_pte(u64 pte)
242{
243 return pte & PT_PAGE_SIZE_MASK;
244}
245
246static int is_writable_pte(unsigned long pte)
247{
248 return pte & PT_WRITABLE_MASK;
249}
250
251static int is_dirty_gpte(unsigned long pte)
252{
253 return pte & PT_DIRTY_MASK;
254}
255
256static int is_rmap_spte(u64 pte)
257{
258 return is_shadow_present_pte(pte);
259}
260
261static int is_last_spte(u64 pte, int level)
262{
263 if (level == PT_PAGE_TABLE_LEVEL)
264 return 1;
265 if (is_large_pte(pte))
266 return 1;
267 return 0;
268}
269
270static pfn_t spte_to_pfn(u64 pte)
271{
272 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
273}
274
275static gfn_t pse36_gfn_delta(u32 gpte)
276{
277 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
278
279 return (gpte & PT32_DIR_PSE36_MASK) << shift;
280}
281
282static void __set_spte(u64 *sptep, u64 spte)
283{
284#ifdef CONFIG_X86_64
285 set_64bit((unsigned long *)sptep, spte);
286#else
287 set_64bit((unsigned long long *)sptep, spte);
288#endif
289}
290
291static u64 __xchg_spte(u64 *sptep, u64 new_spte)
292{
293#ifdef CONFIG_X86_64
294 return xchg(sptep, new_spte);
295#else
296 u64 old_spte;
297
298 do {
299 old_spte = *sptep;
300 } while (cmpxchg64(sptep, old_spte, new_spte) != old_spte);
301
302 return old_spte;
303#endif
304}
305
306static void update_spte(u64 *sptep, u64 new_spte)
307{
308 u64 old_spte;
309
310 if (!shadow_accessed_mask || (new_spte & shadow_accessed_mask)) {
311 __set_spte(sptep, new_spte);
312 } else {
313 old_spte = __xchg_spte(sptep, new_spte);
314 if (old_spte & shadow_accessed_mask)
315 mark_page_accessed(pfn_to_page(spte_to_pfn(old_spte)));
316 }
317}
318
319static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
320 struct kmem_cache *base_cache, int min)
321{
322 void *obj;
323
324 if (cache->nobjs >= min)
325 return 0;
326 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
327 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
328 if (!obj)
329 return -ENOMEM;
330 cache->objects[cache->nobjs++] = obj;
331 }
332 return 0;
333}
334
335static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
336 struct kmem_cache *cache)
337{
338 while (mc->nobjs)
339 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
340}
341
342static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
343 int min)
344{
345 struct page *page;
346
347 if (cache->nobjs >= min)
348 return 0;
349 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
350 page = alloc_page(GFP_KERNEL);
351 if (!page)
352 return -ENOMEM;
353 cache->objects[cache->nobjs++] = page_address(page);
354 }
355 return 0;
356}
357
358static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
359{
360 while (mc->nobjs)
361 free_page((unsigned long)mc->objects[--mc->nobjs]);
362}
363
364static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
365{
366 int r;
367
368 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
369 pte_chain_cache, 4);
370 if (r)
371 goto out;
372 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
373 rmap_desc_cache, 4);
374 if (r)
375 goto out;
376 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
377 if (r)
378 goto out;
379 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
380 mmu_page_header_cache, 4);
381out:
382 return r;
383}
384
385static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
386{
387 mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache, pte_chain_cache);
388 mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, rmap_desc_cache);
389 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
390 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
391 mmu_page_header_cache);
392}
393
394static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
395 size_t size)
396{
397 void *p;
398
399 BUG_ON(!mc->nobjs);
400 p = mc->objects[--mc->nobjs];
401 return p;
402}
403
404static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
405{
406 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
407 sizeof(struct kvm_pte_chain));
408}
409
410static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
411{
412 kmem_cache_free(pte_chain_cache, pc);
413}
414
415static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
416{
417 return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
418 sizeof(struct kvm_rmap_desc));
419}
420
421static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
422{
423 kmem_cache_free(rmap_desc_cache, rd);
424}
425
426static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
427{
428 if (!sp->role.direct)
429 return sp->gfns[index];
430
431 return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
432}
433
434static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
435{
436 if (sp->role.direct)
437 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
438 else
439 sp->gfns[index] = gfn;
440}
441
442/*
443 * Return the pointer to the largepage write count for a given
444 * gfn, handling slots that are not large page aligned.
445 */
446static int *slot_largepage_idx(gfn_t gfn,
447 struct kvm_memory_slot *slot,
448 int level)
449{
450 unsigned long idx;
451
452 idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
453 (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
454 return &slot->lpage_info[level - 2][idx].write_count;
455}
456
457static void account_shadowed(struct kvm *kvm, gfn_t gfn)
458{
459 struct kvm_memory_slot *slot;
460 int *write_count;
461 int i;
462
463 slot = gfn_to_memslot(kvm, gfn);
464 for (i = PT_DIRECTORY_LEVEL;
465 i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
466 write_count = slot_largepage_idx(gfn, slot, i);
467 *write_count += 1;
468 }
469}
470
471static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
472{
473 struct kvm_memory_slot *slot;
474 int *write_count;
475 int i;
476
477 slot = gfn_to_memslot(kvm, gfn);
478 for (i = PT_DIRECTORY_LEVEL;
479 i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
480 write_count = slot_largepage_idx(gfn, slot, i);
481 *write_count -= 1;
482 WARN_ON(*write_count < 0);
483 }
484}
485
486static int has_wrprotected_page(struct kvm *kvm,
487 gfn_t gfn,
488 int level)
489{
490 struct kvm_memory_slot *slot;
491 int *largepage_idx;
492
493 slot = gfn_to_memslot(kvm, gfn);
494 if (slot) {
495 largepage_idx = slot_largepage_idx(gfn, slot, level);
496 return *largepage_idx;
497 }
498
499 return 1;
500}
501
502static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
503{
504 unsigned long page_size;
505 int i, ret = 0;
506
507 page_size = kvm_host_page_size(kvm, gfn);
508
509 for (i = PT_PAGE_TABLE_LEVEL;
510 i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
511 if (page_size >= KVM_HPAGE_SIZE(i))
512 ret = i;
513 else
514 break;
515 }
516
517 return ret;
518}
519
520static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
521{
522 struct kvm_memory_slot *slot;
523 int host_level, level, max_level;
524
525 slot = gfn_to_memslot(vcpu->kvm, large_gfn);
526 if (slot && slot->dirty_bitmap)
527 return PT_PAGE_TABLE_LEVEL;
528
529 host_level = host_mapping_level(vcpu->kvm, large_gfn);
530
531 if (host_level == PT_PAGE_TABLE_LEVEL)
532 return host_level;
533
534 max_level = kvm_x86_ops->get_lpage_level() < host_level ?
535 kvm_x86_ops->get_lpage_level() : host_level;
536
537 for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
538 if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
539 break;
540
541 return level - 1;
542}
543
544/*
545 * Take gfn and return the reverse mapping to it.
546 */
547
548static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
549{
550 struct kvm_memory_slot *slot;
551 unsigned long idx;
552
553 slot = gfn_to_memslot(kvm, gfn);
554 if (likely(level == PT_PAGE_TABLE_LEVEL))
555 return &slot->rmap[gfn - slot->base_gfn];
556
557 idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
558 (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
559
560 return &slot->lpage_info[level - 2][idx].rmap_pde;
561}
562
563/*
564 * Reverse mapping data structures:
565 *
566 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
567 * that points to page_address(page).
568 *
569 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
570 * containing more mappings.
571 *
572 * Returns the number of rmap entries before the spte was added or zero if
573 * the spte was not added.
574 *
575 */
576static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
577{
578 struct kvm_mmu_page *sp;
579 struct kvm_rmap_desc *desc;
580 unsigned long *rmapp;
581 int i, count = 0;
582
583 if (!is_rmap_spte(*spte))
584 return count;
585 sp = page_header(__pa(spte));
586 kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
587 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
588 if (!*rmapp) {
589 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
590 *rmapp = (unsigned long)spte;
591 } else if (!(*rmapp & 1)) {
592 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
593 desc = mmu_alloc_rmap_desc(vcpu);
594 desc->sptes[0] = (u64 *)*rmapp;
595 desc->sptes[1] = spte;
596 *rmapp = (unsigned long)desc | 1;
597 } else {
598 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
599 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
600 while (desc->sptes[RMAP_EXT-1] && desc->more) {
601 desc = desc->more;
602 count += RMAP_EXT;
603 }
604 if (desc->sptes[RMAP_EXT-1]) {
605 desc->more = mmu_alloc_rmap_desc(vcpu);
606 desc = desc->more;
607 }
608 for (i = 0; desc->sptes[i]; ++i)
609 ;
610 desc->sptes[i] = spte;
611 }
612 return count;
613}
614
615static void rmap_desc_remove_entry(unsigned long *rmapp,
616 struct kvm_rmap_desc *desc,
617 int i,
618 struct kvm_rmap_desc *prev_desc)
619{
620 int j;
621
622 for (j = RMAP_EXT - 1; !desc->sptes[j] && j > i; --j)
623 ;
624 desc->sptes[i] = desc->sptes[j];
625 desc->sptes[j] = NULL;
626 if (j != 0)
627 return;
628 if (!prev_desc && !desc->more)
629 *rmapp = (unsigned long)desc->sptes[0];
630 else
631 if (prev_desc)
632 prev_desc->more = desc->more;
633 else
634 *rmapp = (unsigned long)desc->more | 1;
635 mmu_free_rmap_desc(desc);
636}
637
638static void rmap_remove(struct kvm *kvm, u64 *spte)
639{
640 struct kvm_rmap_desc *desc;
641 struct kvm_rmap_desc *prev_desc;
642 struct kvm_mmu_page *sp;
643 gfn_t gfn;
644 unsigned long *rmapp;
645 int i;
646
647 sp = page_header(__pa(spte));
648 gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
649 rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
650 if (!*rmapp) {
651 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
652 BUG();
653 } else if (!(*rmapp & 1)) {
654 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
655 if ((u64 *)*rmapp != spte) {
656 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
657 spte, *spte);
658 BUG();
659 }
660 *rmapp = 0;
661 } else {
662 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
663 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
664 prev_desc = NULL;
665 while (desc) {
666 for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i)
667 if (desc->sptes[i] == spte) {
668 rmap_desc_remove_entry(rmapp,
669 desc, i,
670 prev_desc);
671 return;
672 }
673 prev_desc = desc;
674 desc = desc->more;
675 }
676 pr_err("rmap_remove: %p %llx many->many\n", spte, *spte);
677 BUG();
678 }
679}
680
681static void drop_spte(struct kvm *kvm, u64 *sptep, u64 new_spte)
682{
683 pfn_t pfn;
684 u64 old_spte;
685
686 old_spte = __xchg_spte(sptep, new_spte);
687 if (!is_rmap_spte(old_spte))
688 return;
689 pfn = spte_to_pfn(old_spte);
690 if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
691 kvm_set_pfn_accessed(pfn);
692 if (is_writable_pte(old_spte))
693 kvm_set_pfn_dirty(pfn);
694 rmap_remove(kvm, sptep);
695}
696
697static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
698{
699 struct kvm_rmap_desc *desc;
700 u64 *prev_spte;
701 int i;
702
703 if (!*rmapp)
704 return NULL;
705 else if (!(*rmapp & 1)) {
706 if (!spte)
707 return (u64 *)*rmapp;
708 return NULL;
709 }
710 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
711 prev_spte = NULL;
712 while (desc) {
713 for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i) {
714 if (prev_spte == spte)
715 return desc->sptes[i];
716 prev_spte = desc->sptes[i];
717 }
718 desc = desc->more;
719 }
720 return NULL;
721}
722
723static int rmap_write_protect(struct kvm *kvm, u64 gfn)
724{
725 unsigned long *rmapp;
726 u64 *spte;
727 int i, write_protected = 0;
728
729 rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
730
731 spte = rmap_next(kvm, rmapp, NULL);
732 while (spte) {
733 BUG_ON(!spte);
734 BUG_ON(!(*spte & PT_PRESENT_MASK));
735 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
736 if (is_writable_pte(*spte)) {
737 update_spte(spte, *spte & ~PT_WRITABLE_MASK);
738 write_protected = 1;
739 }
740 spte = rmap_next(kvm, rmapp, spte);
741 }
742 if (write_protected) {
743 pfn_t pfn;
744
745 spte = rmap_next(kvm, rmapp, NULL);
746 pfn = spte_to_pfn(*spte);
747 kvm_set_pfn_dirty(pfn);
748 }
749
750 /* check for huge page mappings */
751 for (i = PT_DIRECTORY_LEVEL;
752 i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
753 rmapp = gfn_to_rmap(kvm, gfn, i);
754 spte = rmap_next(kvm, rmapp, NULL);
755 while (spte) {
756 BUG_ON(!spte);
757 BUG_ON(!(*spte & PT_PRESENT_MASK));
758 BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
759 pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
760 if (is_writable_pte(*spte)) {
761 drop_spte(kvm, spte,
762 shadow_trap_nonpresent_pte);
763 --kvm->stat.lpages;
764 spte = NULL;
765 write_protected = 1;
766 }
767 spte = rmap_next(kvm, rmapp, spte);
768 }
769 }
770
771 return write_protected;
772}
773
774static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
775 unsigned long data)
776{
777 u64 *spte;
778 int need_tlb_flush = 0;
779
780 while ((spte = rmap_next(kvm, rmapp, NULL))) {
781 BUG_ON(!(*spte & PT_PRESENT_MASK));
782 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
783 drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
784 need_tlb_flush = 1;
785 }
786 return need_tlb_flush;
787}
788
789static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
790 unsigned long data)
791{
792 int need_flush = 0;
793 u64 *spte, new_spte, old_spte;
794 pte_t *ptep = (pte_t *)data;
795 pfn_t new_pfn;
796
797 WARN_ON(pte_huge(*ptep));
798 new_pfn = pte_pfn(*ptep);
799 spte = rmap_next(kvm, rmapp, NULL);
800 while (spte) {
801 BUG_ON(!is_shadow_present_pte(*spte));
802 rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", spte, *spte);
803 need_flush = 1;
804 if (pte_write(*ptep)) {
805 drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
806 spte = rmap_next(kvm, rmapp, NULL);
807 } else {
808 new_spte = *spte &~ (PT64_BASE_ADDR_MASK);
809 new_spte |= (u64)new_pfn << PAGE_SHIFT;
810
811 new_spte &= ~PT_WRITABLE_MASK;
812 new_spte &= ~SPTE_HOST_WRITEABLE;
813 new_spte &= ~shadow_accessed_mask;
814 if (is_writable_pte(*spte))
815 kvm_set_pfn_dirty(spte_to_pfn(*spte));
816 old_spte = __xchg_spte(spte, new_spte);
817 if (is_shadow_present_pte(old_spte)
818 && (!shadow_accessed_mask ||
819 old_spte & shadow_accessed_mask))
820 mark_page_accessed(pfn_to_page(spte_to_pfn(old_spte)));
821 spte = rmap_next(kvm, rmapp, spte);
822 }
823 }
824 if (need_flush)
825 kvm_flush_remote_tlbs(kvm);
826
827 return 0;
828}
829
830static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
831 unsigned long data,
832 int (*handler)(struct kvm *kvm, unsigned long *rmapp,
833 unsigned long data))
834{
835 int i, j;
836 int ret;
837 int retval = 0;
838 struct kvm_memslots *slots;
839
840 slots = kvm_memslots(kvm);
841
842 for (i = 0; i < slots->nmemslots; i++) {
843 struct kvm_memory_slot *memslot = &slots->memslots[i];
844 unsigned long start = memslot->userspace_addr;
845 unsigned long end;
846
847 end = start + (memslot->npages << PAGE_SHIFT);
848 if (hva >= start && hva < end) {
849 gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
850
851 ret = handler(kvm, &memslot->rmap[gfn_offset], data);
852
853 for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
854 unsigned long idx;
855 int sh;
856
857 sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j);
858 idx = ((memslot->base_gfn+gfn_offset) >> sh) -
859 (memslot->base_gfn >> sh);
860 ret |= handler(kvm,
861 &memslot->lpage_info[j][idx].rmap_pde,
862 data);
863 }
864 trace_kvm_age_page(hva, memslot, ret);
865 retval |= ret;
866 }
867 }
868
869 return retval;
870}
871
872int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
873{
874 return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
875}
876
877void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
878{
879 kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
880}
881
882static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
883 unsigned long data)
884{
885 u64 *spte;
886 int young = 0;
887
888 /*
889 * Emulate the accessed bit for EPT, by checking if this page has
890 * an EPT mapping, and clearing it if it does. On the next access,
891 * a new EPT mapping will be established.
892 * This has some overhead, but not as much as the cost of swapping
893 * out actively used pages or breaking up actively used hugepages.
894 */
895 if (!shadow_accessed_mask)
896 return kvm_unmap_rmapp(kvm, rmapp, data);
897
898 spte = rmap_next(kvm, rmapp, NULL);
899 while (spte) {
900 int _young;
901 u64 _spte = *spte;
902 BUG_ON(!(_spte & PT_PRESENT_MASK));
903 _young = _spte & PT_ACCESSED_MASK;
904 if (_young) {
905 young = 1;
906 clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
907 }
908 spte = rmap_next(kvm, rmapp, spte);
909 }
910 return young;
911}
912
913#define RMAP_RECYCLE_THRESHOLD 1000
914
915static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
916{
917 unsigned long *rmapp;
918 struct kvm_mmu_page *sp;
919
920 sp = page_header(__pa(spte));
921
922 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
923
924 kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
925 kvm_flush_remote_tlbs(vcpu->kvm);
926}
927
928int kvm_age_hva(struct kvm *kvm, unsigned long hva)
929{
930 return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
931}
932
933#ifdef MMU_DEBUG
934static int is_empty_shadow_page(u64 *spt)
935{
936 u64 *pos;
937 u64 *end;
938
939 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
940 if (is_shadow_present_pte(*pos)) {
941 printk(KERN_ERR "%s: %p %llx\n", __func__,
942 pos, *pos);
943 return 0;
944 }
945 return 1;
946}
947#endif
948
949static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
950{
951 ASSERT(is_empty_shadow_page(sp->spt));
952 hlist_del(&sp->hash_link);
953 list_del(&sp->link);
954 __free_page(virt_to_page(sp->spt));
955 if (!sp->role.direct)
956 __free_page(virt_to_page(sp->gfns));
957 kmem_cache_free(mmu_page_header_cache, sp);
958 ++kvm->arch.n_free_mmu_pages;
959}
960
961static unsigned kvm_page_table_hashfn(gfn_t gfn)
962{
963 return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
964}
965
966static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
967 u64 *parent_pte, int direct)
968{
969 struct kvm_mmu_page *sp;
970
971 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
972 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
973 if (!direct)
974 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache,
975 PAGE_SIZE);
976 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
977 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
978 bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
979 sp->multimapped = 0;
980 sp->parent_pte = parent_pte;
981 --vcpu->kvm->arch.n_free_mmu_pages;
982 return sp;
983}
984
985static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
986 struct kvm_mmu_page *sp, u64 *parent_pte)
987{
988 struct kvm_pte_chain *pte_chain;
989 struct hlist_node *node;
990 int i;
991
992 if (!parent_pte)
993 return;
994 if (!sp->multimapped) {
995 u64 *old = sp->parent_pte;
996
997 if (!old) {
998 sp->parent_pte = parent_pte;
999 return;
1000 }
1001 sp->multimapped = 1;
1002 pte_chain = mmu_alloc_pte_chain(vcpu);
1003 INIT_HLIST_HEAD(&sp->parent_ptes);
1004 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
1005 pte_chain->parent_ptes[0] = old;
1006 }
1007 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
1008 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
1009 continue;
1010 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
1011 if (!pte_chain->parent_ptes[i]) {
1012 pte_chain->parent_ptes[i] = parent_pte;
1013 return;
1014 }
1015 }
1016 pte_chain = mmu_alloc_pte_chain(vcpu);
1017 BUG_ON(!pte_chain);
1018 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
1019 pte_chain->parent_ptes[0] = parent_pte;
1020}
1021
1022static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1023 u64 *parent_pte)
1024{
1025 struct kvm_pte_chain *pte_chain;
1026 struct hlist_node *node;
1027 int i;
1028
1029 if (!sp->multimapped) {
1030 BUG_ON(sp->parent_pte != parent_pte);
1031 sp->parent_pte = NULL;
1032 return;
1033 }
1034 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
1035 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
1036 if (!pte_chain->parent_ptes[i])
1037 break;
1038 if (pte_chain->parent_ptes[i] != parent_pte)
1039 continue;
1040 while (i + 1 < NR_PTE_CHAIN_ENTRIES
1041 && pte_chain->parent_ptes[i + 1]) {
1042 pte_chain->parent_ptes[i]
1043 = pte_chain->parent_ptes[i + 1];
1044 ++i;
1045 }
1046 pte_chain->parent_ptes[i] = NULL;
1047 if (i == 0) {
1048 hlist_del(&pte_chain->link);
1049 mmu_free_pte_chain(pte_chain);
1050 if (hlist_empty(&sp->parent_ptes)) {
1051 sp->multimapped = 0;
1052 sp->parent_pte = NULL;
1053 }
1054 }
1055 return;
1056 }
1057 BUG();
1058}
1059
1060static void mmu_parent_walk(struct kvm_mmu_page *sp, mmu_parent_walk_fn fn)
1061{
1062 struct kvm_pte_chain *pte_chain;
1063 struct hlist_node *node;
1064 struct kvm_mmu_page *parent_sp;
1065 int i;
1066
1067 if (!sp->multimapped && sp->parent_pte) {
1068 parent_sp = page_header(__pa(sp->parent_pte));
1069 fn(parent_sp, sp->parent_pte);
1070 return;
1071 }
1072
1073 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
1074 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
1075 u64 *spte = pte_chain->parent_ptes[i];
1076
1077 if (!spte)
1078 break;
1079 parent_sp = page_header(__pa(spte));
1080 fn(parent_sp, spte);
1081 }
1082}
1083
1084static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte);
1085static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1086{
1087 mmu_parent_walk(sp, mark_unsync);
1088}
1089
1090static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte)
1091{
1092 unsigned int index;
1093
1094 index = spte - sp->spt;
1095 if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1096 return;
1097 if (sp->unsync_children++)
1098 return;
1099 kvm_mmu_mark_parents_unsync(sp);
1100}
1101
1102static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
1103 struct kvm_mmu_page *sp)
1104{
1105 int i;
1106
1107 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1108 sp->spt[i] = shadow_trap_nonpresent_pte;
1109}
1110
1111static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1112 struct kvm_mmu_page *sp, bool clear_unsync)
1113{
1114 return 1;
1115}
1116
1117static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1118{
1119}
1120
1121#define KVM_PAGE_ARRAY_NR 16
1122
1123struct kvm_mmu_pages {
1124 struct mmu_page_and_offset {
1125 struct kvm_mmu_page *sp;
1126 unsigned int idx;
1127 } page[KVM_PAGE_ARRAY_NR];
1128 unsigned int nr;
1129};
1130
1131#define for_each_unsync_children(bitmap, idx) \
1132 for (idx = find_first_bit(bitmap, 512); \
1133 idx < 512; \
1134 idx = find_next_bit(bitmap, 512, idx+1))
1135
1136static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1137 int idx)
1138{
1139 int i;
1140
1141 if (sp->unsync)
1142 for (i=0; i < pvec->nr; i++)
1143 if (pvec->page[i].sp == sp)
1144 return 0;
1145
1146 pvec->page[pvec->nr].sp = sp;
1147 pvec->page[pvec->nr].idx = idx;
1148 pvec->nr++;
1149 return (pvec->nr == KVM_PAGE_ARRAY_NR);
1150}
1151
1152static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1153 struct kvm_mmu_pages *pvec)
1154{
1155 int i, ret, nr_unsync_leaf = 0;
1156
1157 for_each_unsync_children(sp->unsync_child_bitmap, i) {
1158 struct kvm_mmu_page *child;
1159 u64 ent = sp->spt[i];
1160
1161 if (!is_shadow_present_pte(ent) || is_large_pte(ent))
1162 goto clear_child_bitmap;
1163
1164 child = page_header(ent & PT64_BASE_ADDR_MASK);
1165
1166 if (child->unsync_children) {
1167 if (mmu_pages_add(pvec, child, i))
1168 return -ENOSPC;
1169
1170 ret = __mmu_unsync_walk(child, pvec);
1171 if (!ret)
1172 goto clear_child_bitmap;
1173 else if (ret > 0)
1174 nr_unsync_leaf += ret;
1175 else
1176 return ret;
1177 } else if (child->unsync) {
1178 nr_unsync_leaf++;
1179 if (mmu_pages_add(pvec, child, i))
1180 return -ENOSPC;
1181 } else
1182 goto clear_child_bitmap;
1183
1184 continue;
1185
1186clear_child_bitmap:
1187 __clear_bit(i, sp->unsync_child_bitmap);
1188 sp->unsync_children--;
1189 WARN_ON((int)sp->unsync_children < 0);
1190 }
1191
1192
1193 return nr_unsync_leaf;
1194}
1195
1196static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1197 struct kvm_mmu_pages *pvec)
1198{
1199 if (!sp->unsync_children)
1200 return 0;
1201
1202 mmu_pages_add(pvec, sp, 0);
1203 return __mmu_unsync_walk(sp, pvec);
1204}
1205
1206static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1207{
1208 WARN_ON(!sp->unsync);
1209 trace_kvm_mmu_sync_page(sp);
1210 sp->unsync = 0;
1211 --kvm->stat.mmu_unsync;
1212}
1213
1214static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1215 struct list_head *invalid_list);
1216static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1217 struct list_head *invalid_list);
1218
1219#define for_each_gfn_sp(kvm, sp, gfn, pos) \
1220 hlist_for_each_entry(sp, pos, \
1221 &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link) \
1222 if ((sp)->gfn != (gfn)) {} else
1223
1224#define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos) \
1225 hlist_for_each_entry(sp, pos, \
1226 &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link) \
1227 if ((sp)->gfn != (gfn) || (sp)->role.direct || \
1228 (sp)->role.invalid) {} else
1229
1230/* @sp->gfn should be write-protected at the call site */
1231static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1232 struct list_head *invalid_list, bool clear_unsync)
1233{
1234 if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1235 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1236 return 1;
1237 }
1238
1239 if (clear_unsync)
1240 kvm_unlink_unsync_page(vcpu->kvm, sp);
1241
1242 if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) {
1243 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1244 return 1;
1245 }
1246
1247 kvm_mmu_flush_tlb(vcpu);
1248 return 0;
1249}
1250
1251static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
1252 struct kvm_mmu_page *sp)
1253{
1254 LIST_HEAD(invalid_list);
1255 int ret;
1256
1257 ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1258 if (ret)
1259 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1260
1261 return ret;
1262}
1263
1264static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1265 struct list_head *invalid_list)
1266{
1267 return __kvm_sync_page(vcpu, sp, invalid_list, true);
1268}
1269
1270/* @gfn should be write-protected at the call site */
1271static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
1272{
1273 struct kvm_mmu_page *s;
1274 struct hlist_node *node;
1275 LIST_HEAD(invalid_list);
1276 bool flush = false;
1277
1278 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1279 if (!s->unsync)
1280 continue;
1281
1282 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1283 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1284 (vcpu->arch.mmu.sync_page(vcpu, s, true))) {
1285 kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1286 continue;
1287 }
1288 kvm_unlink_unsync_page(vcpu->kvm, s);
1289 flush = true;
1290 }
1291
1292 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1293 if (flush)
1294 kvm_mmu_flush_tlb(vcpu);
1295}
1296
1297struct mmu_page_path {
1298 struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1299 unsigned int idx[PT64_ROOT_LEVEL-1];
1300};
1301
1302#define for_each_sp(pvec, sp, parents, i) \
1303 for (i = mmu_pages_next(&pvec, &parents, -1), \
1304 sp = pvec.page[i].sp; \
1305 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
1306 i = mmu_pages_next(&pvec, &parents, i))
1307
1308static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1309 struct mmu_page_path *parents,
1310 int i)
1311{
1312 int n;
1313
1314 for (n = i+1; n < pvec->nr; n++) {
1315 struct kvm_mmu_page *sp = pvec->page[n].sp;
1316
1317 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1318 parents->idx[0] = pvec->page[n].idx;
1319 return n;
1320 }
1321
1322 parents->parent[sp->role.level-2] = sp;
1323 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1324 }
1325
1326 return n;
1327}
1328
1329static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1330{
1331 struct kvm_mmu_page *sp;
1332 unsigned int level = 0;
1333
1334 do {
1335 unsigned int idx = parents->idx[level];
1336
1337 sp = parents->parent[level];
1338 if (!sp)
1339 return;
1340
1341 --sp->unsync_children;
1342 WARN_ON((int)sp->unsync_children < 0);
1343 __clear_bit(idx, sp->unsync_child_bitmap);
1344 level++;
1345 } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1346}
1347
1348static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
1349 struct mmu_page_path *parents,
1350 struct kvm_mmu_pages *pvec)
1351{
1352 parents->parent[parent->role.level-1] = NULL;
1353 pvec->nr = 0;
1354}
1355
1356static void mmu_sync_children(struct kvm_vcpu *vcpu,
1357 struct kvm_mmu_page *parent)
1358{
1359 int i;
1360 struct kvm_mmu_page *sp;
1361 struct mmu_page_path parents;
1362 struct kvm_mmu_pages pages;
1363 LIST_HEAD(invalid_list);
1364
1365 kvm_mmu_pages_init(parent, &parents, &pages);
1366 while (mmu_unsync_walk(parent, &pages)) {
1367 int protected = 0;
1368
1369 for_each_sp(pages, sp, parents, i)
1370 protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
1371
1372 if (protected)
1373 kvm_flush_remote_tlbs(vcpu->kvm);
1374
1375 for_each_sp(pages, sp, parents, i) {
1376 kvm_sync_page(vcpu, sp, &invalid_list);
1377 mmu_pages_clear_parents(&parents);
1378 }
1379 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1380 cond_resched_lock(&vcpu->kvm->mmu_lock);
1381 kvm_mmu_pages_init(parent, &parents, &pages);
1382 }
1383}
1384
1385static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
1386 gfn_t gfn,
1387 gva_t gaddr,
1388 unsigned level,
1389 int direct,
1390 unsigned access,
1391 u64 *parent_pte)
1392{
1393 union kvm_mmu_page_role role;
1394 unsigned quadrant;
1395 struct kvm_mmu_page *sp;
1396 struct hlist_node *node;
1397 bool need_sync = false;
1398
1399 role = vcpu->arch.mmu.base_role;
1400 role.level = level;
1401 role.direct = direct;
1402 if (role.direct)
1403 role.cr4_pae = 0;
1404 role.access = access;
1405 if (!tdp_enabled && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1406 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
1407 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
1408 role.quadrant = quadrant;
1409 }
1410 for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
1411 if (!need_sync && sp->unsync)
1412 need_sync = true;
1413
1414 if (sp->role.word != role.word)
1415 continue;
1416
1417 if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
1418 break;
1419
1420 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1421 if (sp->unsync_children) {
1422 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1423 kvm_mmu_mark_parents_unsync(sp);
1424 } else if (sp->unsync)
1425 kvm_mmu_mark_parents_unsync(sp);
1426
1427 trace_kvm_mmu_get_page(sp, false);
1428 return sp;
1429 }
1430 ++vcpu->kvm->stat.mmu_cache_miss;
1431 sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
1432 if (!sp)
1433 return sp;
1434 sp->gfn = gfn;
1435 sp->role = role;
1436 hlist_add_head(&sp->hash_link,
1437 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
1438 if (!direct) {
1439 if (rmap_write_protect(vcpu->kvm, gfn))
1440 kvm_flush_remote_tlbs(vcpu->kvm);
1441 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
1442 kvm_sync_pages(vcpu, gfn);
1443
1444 account_shadowed(vcpu->kvm, gfn);
1445 }
1446 if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
1447 vcpu->arch.mmu.prefetch_page(vcpu, sp);
1448 else
1449 nonpaging_prefetch_page(vcpu, sp);
1450 trace_kvm_mmu_get_page(sp, true);
1451 return sp;
1452}
1453
1454static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
1455 struct kvm_vcpu *vcpu, u64 addr)
1456{
1457 iterator->addr = addr;
1458 iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
1459 iterator->level = vcpu->arch.mmu.shadow_root_level;
1460 if (iterator->level == PT32E_ROOT_LEVEL) {
1461 iterator->shadow_addr
1462 = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
1463 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
1464 --iterator->level;
1465 if (!iterator->shadow_addr)
1466 iterator->level = 0;
1467 }
1468}
1469
1470static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
1471{
1472 if (iterator->level < PT_PAGE_TABLE_LEVEL)
1473 return false;
1474
1475 if (iterator->level == PT_PAGE_TABLE_LEVEL)
1476 if (is_large_pte(*iterator->sptep))
1477 return false;
1478
1479 iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
1480 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
1481 return true;
1482}
1483
1484static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
1485{
1486 iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
1487 --iterator->level;
1488}
1489
1490static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
1491{
1492 u64 spte;
1493
1494 spte = __pa(sp->spt)
1495 | PT_PRESENT_MASK | PT_ACCESSED_MASK
1496 | PT_WRITABLE_MASK | PT_USER_MASK;
1497 __set_spte(sptep, spte);
1498}
1499
1500static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1501{
1502 if (is_large_pte(*sptep)) {
1503 drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
1504 kvm_flush_remote_tlbs(vcpu->kvm);
1505 }
1506}
1507
1508static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
1509 unsigned direct_access)
1510{
1511 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
1512 struct kvm_mmu_page *child;
1513
1514 /*
1515 * For the direct sp, if the guest pte's dirty bit
1516 * changed form clean to dirty, it will corrupt the
1517 * sp's access: allow writable in the read-only sp,
1518 * so we should update the spte at this point to get
1519 * a new sp with the correct access.
1520 */
1521 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
1522 if (child->role.access == direct_access)
1523 return;
1524
1525 mmu_page_remove_parent_pte(child, sptep);
1526 __set_spte(sptep, shadow_trap_nonpresent_pte);
1527 kvm_flush_remote_tlbs(vcpu->kvm);
1528 }
1529}
1530
1531static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1532 struct kvm_mmu_page *sp)
1533{
1534 unsigned i;
1535 u64 *pt;
1536 u64 ent;
1537
1538 pt = sp->spt;
1539
1540 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1541 ent = pt[i];
1542
1543 if (is_shadow_present_pte(ent)) {
1544 if (!is_last_spte(ent, sp->role.level)) {
1545 ent &= PT64_BASE_ADDR_MASK;
1546 mmu_page_remove_parent_pte(page_header(ent),
1547 &pt[i]);
1548 } else {
1549 if (is_large_pte(ent))
1550 --kvm->stat.lpages;
1551 drop_spte(kvm, &pt[i],
1552 shadow_trap_nonpresent_pte);
1553 }
1554 }
1555 pt[i] = shadow_trap_nonpresent_pte;
1556 }
1557}
1558
1559static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
1560{
1561 mmu_page_remove_parent_pte(sp, parent_pte);
1562}
1563
1564static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
1565{
1566 int i;
1567 struct kvm_vcpu *vcpu;
1568
1569 kvm_for_each_vcpu(i, vcpu, kvm)
1570 vcpu->arch.last_pte_updated = NULL;
1571}
1572
1573static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
1574{
1575 u64 *parent_pte;
1576
1577 while (sp->multimapped || sp->parent_pte) {
1578 if (!sp->multimapped)
1579 parent_pte = sp->parent_pte;
1580 else {
1581 struct kvm_pte_chain *chain;
1582
1583 chain = container_of(sp->parent_ptes.first,
1584 struct kvm_pte_chain, link);
1585 parent_pte = chain->parent_ptes[0];
1586 }
1587 BUG_ON(!parent_pte);
1588 kvm_mmu_put_page(sp, parent_pte);
1589 __set_spte(parent_pte, shadow_trap_nonpresent_pte);
1590 }
1591}
1592
1593static int mmu_zap_unsync_children(struct kvm *kvm,
1594 struct kvm_mmu_page *parent,
1595 struct list_head *invalid_list)
1596{
1597 int i, zapped = 0;
1598 struct mmu_page_path parents;
1599 struct kvm_mmu_pages pages;
1600
1601 if (parent->role.level == PT_PAGE_TABLE_LEVEL)
1602 return 0;
1603
1604 kvm_mmu_pages_init(parent, &parents, &pages);
1605 while (mmu_unsync_walk(parent, &pages)) {
1606 struct kvm_mmu_page *sp;
1607
1608 for_each_sp(pages, sp, parents, i) {
1609 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
1610 mmu_pages_clear_parents(&parents);
1611 zapped++;
1612 }
1613 kvm_mmu_pages_init(parent, &parents, &pages);
1614 }
1615
1616 return zapped;
1617}
1618
1619static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1620 struct list_head *invalid_list)
1621{
1622 int ret;
1623
1624 trace_kvm_mmu_prepare_zap_page(sp);
1625 ++kvm->stat.mmu_shadow_zapped;
1626 ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
1627 kvm_mmu_page_unlink_children(kvm, sp);
1628 kvm_mmu_unlink_parents(kvm, sp);
1629 if (!sp->role.invalid && !sp->role.direct)
1630 unaccount_shadowed(kvm, sp->gfn);
1631 if (sp->unsync)
1632 kvm_unlink_unsync_page(kvm, sp);
1633 if (!sp->root_count) {
1634 /* Count self */
1635 ret++;
1636 list_move(&sp->link, invalid_list);
1637 } else {
1638 list_move(&sp->link, &kvm->arch.active_mmu_pages);
1639 kvm_reload_remote_mmus(kvm);
1640 }
1641
1642 sp->role.invalid = 1;
1643 kvm_mmu_reset_last_pte_updated(kvm);
1644 return ret;
1645}
1646
1647static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1648 struct list_head *invalid_list)
1649{
1650 struct kvm_mmu_page *sp;
1651
1652 if (list_empty(invalid_list))
1653 return;
1654
1655 kvm_flush_remote_tlbs(kvm);
1656
1657 do {
1658 sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
1659 WARN_ON(!sp->role.invalid || sp->root_count);
1660 kvm_mmu_free_page(kvm, sp);
1661 } while (!list_empty(invalid_list));
1662
1663}
1664
1665/*
1666 * Changing the number of mmu pages allocated to the vm
1667 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
1668 */
1669void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
1670{
1671 int used_pages;
1672 LIST_HEAD(invalid_list);
1673
1674 used_pages = kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages;
1675 used_pages = max(0, used_pages);
1676
1677 /*
1678 * If we set the number of mmu pages to be smaller be than the
1679 * number of actived pages , we must to free some mmu pages before we
1680 * change the value
1681 */
1682
1683 if (used_pages > kvm_nr_mmu_pages) {
1684 while (used_pages > kvm_nr_mmu_pages &&
1685 !list_empty(&kvm->arch.active_mmu_pages)) {
1686 struct kvm_mmu_page *page;
1687
1688 page = container_of(kvm->arch.active_mmu_pages.prev,
1689 struct kvm_mmu_page, link);
1690 used_pages -= kvm_mmu_prepare_zap_page(kvm, page,
1691 &invalid_list);
1692 }
1693 kvm_mmu_commit_zap_page(kvm, &invalid_list);
1694 kvm_nr_mmu_pages = used_pages;
1695 kvm->arch.n_free_mmu_pages = 0;
1696 }
1697 else
1698 kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
1699 - kvm->arch.n_alloc_mmu_pages;
1700
1701 kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
1702}
1703
1704static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1705{
1706 struct kvm_mmu_page *sp;
1707 struct hlist_node *node;
1708 LIST_HEAD(invalid_list);
1709 int r;
1710
1711 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1712 r = 0;
1713
1714 for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1715 pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
1716 sp->role.word);
1717 r = 1;
1718 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1719 }
1720 kvm_mmu_commit_zap_page(kvm, &invalid_list);
1721 return r;
1722}
1723
1724static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1725{
1726 struct kvm_mmu_page *sp;
1727 struct hlist_node *node;
1728 LIST_HEAD(invalid_list);
1729
1730 for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1731 pgprintk("%s: zap %lx %x\n",
1732 __func__, gfn, sp->role.word);
1733 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1734 }
1735 kvm_mmu_commit_zap_page(kvm, &invalid_list);
1736}
1737
1738static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
1739{
1740 int slot = memslot_id(kvm, gfn);
1741 struct kvm_mmu_page *sp = page_header(__pa(pte));
1742
1743 __set_bit(slot, sp->slot_bitmap);
1744}
1745
1746static void mmu_convert_notrap(struct kvm_mmu_page *sp)
1747{
1748 int i;
1749 u64 *pt = sp->spt;
1750
1751 if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
1752 return;
1753
1754 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1755 if (pt[i] == shadow_notrap_nonpresent_pte)
1756 __set_spte(&pt[i], shadow_trap_nonpresent_pte);
1757 }
1758}
1759
1760/*
1761 * The function is based on mtrr_type_lookup() in
1762 * arch/x86/kernel/cpu/mtrr/generic.c
1763 */
1764static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
1765 u64 start, u64 end)
1766{
1767 int i;
1768 u64 base, mask;
1769 u8 prev_match, curr_match;
1770 int num_var_ranges = KVM_NR_VAR_MTRR;
1771
1772 if (!mtrr_state->enabled)
1773 return 0xFF;
1774
1775 /* Make end inclusive end, instead of exclusive */
1776 end--;
1777
1778 /* Look in fixed ranges. Just return the type as per start */
1779 if (mtrr_state->have_fixed && (start < 0x100000)) {
1780 int idx;
1781
1782 if (start < 0x80000) {
1783 idx = 0;
1784 idx += (start >> 16);
1785 return mtrr_state->fixed_ranges[idx];
1786 } else if (start < 0xC0000) {
1787 idx = 1 * 8;
1788 idx += ((start - 0x80000) >> 14);
1789 return mtrr_state->fixed_ranges[idx];
1790 } else if (start < 0x1000000) {
1791 idx = 3 * 8;
1792 idx += ((start - 0xC0000) >> 12);
1793 return mtrr_state->fixed_ranges[idx];
1794 }
1795 }
1796
1797 /*
1798 * Look in variable ranges
1799 * Look of multiple ranges matching this address and pick type
1800 * as per MTRR precedence
1801 */
1802 if (!(mtrr_state->enabled & 2))
1803 return mtrr_state->def_type;
1804
1805 prev_match = 0xFF;
1806 for (i = 0; i < num_var_ranges; ++i) {
1807 unsigned short start_state, end_state;
1808
1809 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
1810 continue;
1811
1812 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
1813 (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
1814 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
1815 (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
1816
1817 start_state = ((start & mask) == (base & mask));
1818 end_state = ((end & mask) == (base & mask));
1819 if (start_state != end_state)
1820 return 0xFE;
1821
1822 if ((start & mask) != (base & mask))
1823 continue;
1824
1825 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
1826 if (prev_match == 0xFF) {
1827 prev_match = curr_match;
1828 continue;
1829 }
1830
1831 if (prev_match == MTRR_TYPE_UNCACHABLE ||
1832 curr_match == MTRR_TYPE_UNCACHABLE)
1833 return MTRR_TYPE_UNCACHABLE;
1834
1835 if ((prev_match == MTRR_TYPE_WRBACK &&
1836 curr_match == MTRR_TYPE_WRTHROUGH) ||
1837 (prev_match == MTRR_TYPE_WRTHROUGH &&
1838 curr_match == MTRR_TYPE_WRBACK)) {
1839 prev_match = MTRR_TYPE_WRTHROUGH;
1840 curr_match = MTRR_TYPE_WRTHROUGH;
1841 }
1842
1843 if (prev_match != curr_match)
1844 return MTRR_TYPE_UNCACHABLE;
1845 }
1846
1847 if (prev_match != 0xFF)
1848 return prev_match;
1849
1850 return mtrr_state->def_type;
1851}
1852
1853u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
1854{
1855 u8 mtrr;
1856
1857 mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
1858 (gfn << PAGE_SHIFT) + PAGE_SIZE);
1859 if (mtrr == 0xfe || mtrr == 0xff)
1860 mtrr = MTRR_TYPE_WRBACK;
1861 return mtrr;
1862}
1863EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
1864
1865static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1866{
1867 trace_kvm_mmu_unsync_page(sp);
1868 ++vcpu->kvm->stat.mmu_unsync;
1869 sp->unsync = 1;
1870
1871 kvm_mmu_mark_parents_unsync(sp);
1872 mmu_convert_notrap(sp);
1873}
1874
1875static void kvm_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn)
1876{
1877 struct kvm_mmu_page *s;
1878 struct hlist_node *node;
1879
1880 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1881 if (s->unsync)
1882 continue;
1883 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1884 __kvm_unsync_page(vcpu, s);
1885 }
1886}
1887
1888static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
1889 bool can_unsync)
1890{
1891 struct kvm_mmu_page *s;
1892 struct hlist_node *node;
1893 bool need_unsync = false;
1894
1895 for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1896 if (!can_unsync)
1897 return 1;
1898
1899 if (s->role.level != PT_PAGE_TABLE_LEVEL)
1900 return 1;
1901
1902 if (!need_unsync && !s->unsync) {
1903 if (!oos_shadow)
1904 return 1;
1905 need_unsync = true;
1906 }
1907 }
1908 if (need_unsync)
1909 kvm_unsync_pages(vcpu, gfn);
1910 return 0;
1911}
1912
1913static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
1914 unsigned pte_access, int user_fault,
1915 int write_fault, int dirty, int level,
1916 gfn_t gfn, pfn_t pfn, bool speculative,
1917 bool can_unsync, bool reset_host_protection)
1918{
1919 u64 spte;
1920 int ret = 0;
1921
1922 /*
1923 * We don't set the accessed bit, since we sometimes want to see
1924 * whether the guest actually used the pte (in order to detect
1925 * demand paging).
1926 */
1927 spte = shadow_base_present_pte | shadow_dirty_mask;
1928 if (!speculative)
1929 spte |= shadow_accessed_mask;
1930 if (!dirty)
1931 pte_access &= ~ACC_WRITE_MASK;
1932 if (pte_access & ACC_EXEC_MASK)
1933 spte |= shadow_x_mask;
1934 else
1935 spte |= shadow_nx_mask;
1936 if (pte_access & ACC_USER_MASK)
1937 spte |= shadow_user_mask;
1938 if (level > PT_PAGE_TABLE_LEVEL)
1939 spte |= PT_PAGE_SIZE_MASK;
1940 if (tdp_enabled)
1941 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
1942 kvm_is_mmio_pfn(pfn));
1943
1944 if (reset_host_protection)
1945 spte |= SPTE_HOST_WRITEABLE;
1946
1947 spte |= (u64)pfn << PAGE_SHIFT;
1948
1949 if ((pte_access & ACC_WRITE_MASK)
1950 || (!tdp_enabled && write_fault && !is_write_protection(vcpu)
1951 && !user_fault)) {
1952
1953 if (level > PT_PAGE_TABLE_LEVEL &&
1954 has_wrprotected_page(vcpu->kvm, gfn, level)) {
1955 ret = 1;
1956 drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
1957 goto done;
1958 }
1959
1960 spte |= PT_WRITABLE_MASK;
1961
1962 if (!tdp_enabled && !(pte_access & ACC_WRITE_MASK))
1963 spte &= ~PT_USER_MASK;
1964
1965 /*
1966 * Optimization: for pte sync, if spte was writable the hash
1967 * lookup is unnecessary (and expensive). Write protection
1968 * is responsibility of mmu_get_page / kvm_sync_page.
1969 * Same reasoning can be applied to dirty page accounting.
1970 */
1971 if (!can_unsync && is_writable_pte(*sptep))
1972 goto set_pte;
1973
1974 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
1975 pgprintk("%s: found shadow page for %lx, marking ro\n",
1976 __func__, gfn);
1977 ret = 1;
1978 pte_access &= ~ACC_WRITE_MASK;
1979 if (is_writable_pte(spte))
1980 spte &= ~PT_WRITABLE_MASK;
1981 }
1982 }
1983
1984 if (pte_access & ACC_WRITE_MASK)
1985 mark_page_dirty(vcpu->kvm, gfn);
1986
1987set_pte:
1988 update_spte(sptep, spte);
1989done:
1990 return ret;
1991}
1992
1993static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
1994 unsigned pt_access, unsigned pte_access,
1995 int user_fault, int write_fault, int dirty,
1996 int *ptwrite, int level, gfn_t gfn,
1997 pfn_t pfn, bool speculative,
1998 bool reset_host_protection)
1999{
2000 int was_rmapped = 0;
2001 int was_writable = is_writable_pte(*sptep);
2002 int rmap_count;
2003
2004 pgprintk("%s: spte %llx access %x write_fault %d"
2005 " user_fault %d gfn %lx\n",
2006 __func__, *sptep, pt_access,
2007 write_fault, user_fault, gfn);
2008
2009 if (is_rmap_spte(*sptep)) {
2010 /*
2011 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2012 * the parent of the now unreachable PTE.
2013 */
2014 if (level > PT_PAGE_TABLE_LEVEL &&
2015 !is_large_pte(*sptep)) {
2016 struct kvm_mmu_page *child;
2017 u64 pte = *sptep;
2018
2019 child = page_header(pte & PT64_BASE_ADDR_MASK);
2020 mmu_page_remove_parent_pte(child, sptep);
2021 __set_spte(sptep, shadow_trap_nonpresent_pte);
2022 kvm_flush_remote_tlbs(vcpu->kvm);
2023 } else if (pfn != spte_to_pfn(*sptep)) {
2024 pgprintk("hfn old %lx new %lx\n",
2025 spte_to_pfn(*sptep), pfn);
2026 drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
2027 kvm_flush_remote_tlbs(vcpu->kvm);
2028 } else
2029 was_rmapped = 1;
2030 }
2031
2032 if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
2033 dirty, level, gfn, pfn, speculative, true,
2034 reset_host_protection)) {
2035 if (write_fault)
2036 *ptwrite = 1;
2037 kvm_mmu_flush_tlb(vcpu);
2038 }
2039
2040 pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2041 pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
2042 is_large_pte(*sptep)? "2MB" : "4kB",
2043 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2044 *sptep, sptep);
2045 if (!was_rmapped && is_large_pte(*sptep))
2046 ++vcpu->kvm->stat.lpages;
2047
2048 page_header_update_slot(vcpu->kvm, sptep, gfn);
2049 if (!was_rmapped) {
2050 rmap_count = rmap_add(vcpu, sptep, gfn);
2051 kvm_release_pfn_clean(pfn);
2052 if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2053 rmap_recycle(vcpu, sptep, gfn);
2054 } else {
2055 if (was_writable)
2056 kvm_release_pfn_dirty(pfn);
2057 else
2058 kvm_release_pfn_clean(pfn);
2059 }
2060 if (speculative) {
2061 vcpu->arch.last_pte_updated = sptep;
2062 vcpu->arch.last_pte_gfn = gfn;
2063 }
2064}
2065
2066static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
2067{
2068}
2069
2070static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2071 int level, gfn_t gfn, pfn_t pfn)
2072{
2073 struct kvm_shadow_walk_iterator iterator;
2074 struct kvm_mmu_page *sp;
2075 int pt_write = 0;
2076 gfn_t pseudo_gfn;
2077
2078 for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2079 if (iterator.level == level) {
2080 mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
2081 0, write, 1, &pt_write,
2082 level, gfn, pfn, false, true);
2083 ++vcpu->stat.pf_fixed;
2084 break;
2085 }
2086
2087 if (*iterator.sptep == shadow_trap_nonpresent_pte) {
2088 u64 base_addr = iterator.addr;
2089
2090 base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2091 pseudo_gfn = base_addr >> PAGE_SHIFT;
2092 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2093 iterator.level - 1,
2094 1, ACC_ALL, iterator.sptep);
2095 if (!sp) {
2096 pgprintk("nonpaging_map: ENOMEM\n");
2097 kvm_release_pfn_clean(pfn);
2098 return -ENOMEM;
2099 }
2100
2101 __set_spte(iterator.sptep,
2102 __pa(sp->spt)
2103 | PT_PRESENT_MASK | PT_WRITABLE_MASK
2104 | shadow_user_mask | shadow_x_mask);
2105 }
2106 }
2107 return pt_write;
2108}
2109
2110static void kvm_send_hwpoison_signal(struct kvm *kvm, gfn_t gfn)
2111{
2112 char buf[1];
2113 void __user *hva;
2114 int r;
2115
2116 /* Touch the page, so send SIGBUS */
2117 hva = (void __user *)gfn_to_hva(kvm, gfn);
2118 r = copy_from_user(buf, hva, 1);
2119}
2120
2121static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn)
2122{
2123 kvm_release_pfn_clean(pfn);
2124 if (is_hwpoison_pfn(pfn)) {
2125 kvm_send_hwpoison_signal(kvm, gfn);
2126 return 0;
2127 } else if (is_fault_pfn(pfn))
2128 return -EFAULT;
2129
2130 return 1;
2131}
2132
2133static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
2134{
2135 int r;
2136 int level;
2137 pfn_t pfn;
2138 unsigned long mmu_seq;
2139
2140 level = mapping_level(vcpu, gfn);
2141
2142 /*
2143 * This path builds a PAE pagetable - so we can map 2mb pages at
2144 * maximum. Therefore check if the level is larger than that.
2145 */
2146 if (level > PT_DIRECTORY_LEVEL)
2147 level = PT_DIRECTORY_LEVEL;
2148
2149 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
2150
2151 mmu_seq = vcpu->kvm->mmu_notifier_seq;
2152 smp_rmb();
2153 pfn = gfn_to_pfn(vcpu->kvm, gfn);
2154
2155 /* mmio */
2156 if (is_error_pfn(pfn))
2157 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
2158
2159 spin_lock(&vcpu->kvm->mmu_lock);
2160 if (mmu_notifier_retry(vcpu, mmu_seq))
2161 goto out_unlock;
2162 kvm_mmu_free_some_pages(vcpu);
2163 r = __direct_map(vcpu, v, write, level, gfn, pfn);
2164 spin_unlock(&vcpu->kvm->mmu_lock);
2165
2166
2167 return r;
2168
2169out_unlock:
2170 spin_unlock(&vcpu->kvm->mmu_lock);
2171 kvm_release_pfn_clean(pfn);
2172 return 0;
2173}
2174
2175
2176static void mmu_free_roots(struct kvm_vcpu *vcpu)
2177{
2178 int i;
2179 struct kvm_mmu_page *sp;
2180 LIST_HEAD(invalid_list);
2181
2182 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2183 return;
2184 spin_lock(&vcpu->kvm->mmu_lock);
2185 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2186 hpa_t root = vcpu->arch.mmu.root_hpa;
2187
2188 sp = page_header(root);
2189 --sp->root_count;
2190 if (!sp->root_count && sp->role.invalid) {
2191 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
2192 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2193 }
2194 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2195 spin_unlock(&vcpu->kvm->mmu_lock);
2196 return;
2197 }
2198 for (i = 0; i < 4; ++i) {
2199 hpa_t root = vcpu->arch.mmu.pae_root[i];
2200
2201 if (root) {
2202 root &= PT64_BASE_ADDR_MASK;
2203 sp = page_header(root);
2204 --sp->root_count;
2205 if (!sp->root_count && sp->role.invalid)
2206 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2207 &invalid_list);
2208 }
2209 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2210 }
2211 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2212 spin_unlock(&vcpu->kvm->mmu_lock);
2213 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2214}
2215
2216static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
2217{
2218 int ret = 0;
2219
2220 if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
2221 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2222 ret = 1;
2223 }
2224
2225 return ret;
2226}
2227
2228static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
2229{
2230 int i;
2231 gfn_t root_gfn;
2232 struct kvm_mmu_page *sp;
2233 int direct = 0;
2234 u64 pdptr;
2235
2236 root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
2237
2238 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2239 hpa_t root = vcpu->arch.mmu.root_hpa;
2240
2241 ASSERT(!VALID_PAGE(root));
2242 if (mmu_check_root(vcpu, root_gfn))
2243 return 1;
2244 if (tdp_enabled) {
2245 direct = 1;
2246 root_gfn = 0;
2247 }
2248 spin_lock(&vcpu->kvm->mmu_lock);
2249 kvm_mmu_free_some_pages(vcpu);
2250 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
2251 PT64_ROOT_LEVEL, direct,
2252 ACC_ALL, NULL);
2253 root = __pa(sp->spt);
2254 ++sp->root_count;
2255 spin_unlock(&vcpu->kvm->mmu_lock);
2256 vcpu->arch.mmu.root_hpa = root;
2257 return 0;
2258 }
2259 direct = !is_paging(vcpu);
2260 for (i = 0; i < 4; ++i) {
2261 hpa_t root = vcpu->arch.mmu.pae_root[i];
2262
2263 ASSERT(!VALID_PAGE(root));
2264 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
2265 pdptr = kvm_pdptr_read(vcpu, i);
2266 if (!is_present_gpte(pdptr)) {
2267 vcpu->arch.mmu.pae_root[i] = 0;
2268 continue;
2269 }
2270 root_gfn = pdptr >> PAGE_SHIFT;
2271 } else if (vcpu->arch.mmu.root_level == 0)
2272 root_gfn = 0;
2273 if (mmu_check_root(vcpu, root_gfn))
2274 return 1;
2275 if (tdp_enabled) {
2276 direct = 1;
2277 root_gfn = i << 30;
2278 }
2279 spin_lock(&vcpu->kvm->mmu_lock);
2280 kvm_mmu_free_some_pages(vcpu);
2281 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
2282 PT32_ROOT_LEVEL, direct,
2283 ACC_ALL, NULL);
2284 root = __pa(sp->spt);
2285 ++sp->root_count;
2286 spin_unlock(&vcpu->kvm->mmu_lock);
2287
2288 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
2289 }
2290 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2291 return 0;
2292}
2293
2294static void mmu_sync_roots(struct kvm_vcpu *vcpu)
2295{
2296 int i;
2297 struct kvm_mmu_page *sp;
2298
2299 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2300 return;
2301 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2302 hpa_t root = vcpu->arch.mmu.root_hpa;
2303 sp = page_header(root);
2304 mmu_sync_children(vcpu, sp);
2305 return;
2306 }
2307 for (i = 0; i < 4; ++i) {
2308 hpa_t root = vcpu->arch.mmu.pae_root[i];
2309
2310 if (root && VALID_PAGE(root)) {
2311 root &= PT64_BASE_ADDR_MASK;
2312 sp = page_header(root);
2313 mmu_sync_children(vcpu, sp);
2314 }
2315 }
2316}
2317
2318void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
2319{
2320 spin_lock(&vcpu->kvm->mmu_lock);
2321 mmu_sync_roots(vcpu);
2322 spin_unlock(&vcpu->kvm->mmu_lock);
2323}
2324
2325static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
2326 u32 access, u32 *error)
2327{
2328 if (error)
2329 *error = 0;
2330 return vaddr;
2331}
2332
2333static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
2334 u32 error_code)
2335{
2336 gfn_t gfn;
2337 int r;
2338
2339 pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
2340 r = mmu_topup_memory_caches(vcpu);
2341 if (r)
2342 return r;
2343
2344 ASSERT(vcpu);
2345 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2346
2347 gfn = gva >> PAGE_SHIFT;
2348
2349 return nonpaging_map(vcpu, gva & PAGE_MASK,
2350 error_code & PFERR_WRITE_MASK, gfn);
2351}
2352
2353static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
2354 u32 error_code)
2355{
2356 pfn_t pfn;
2357 int r;
2358 int level;
2359 gfn_t gfn = gpa >> PAGE_SHIFT;
2360 unsigned long mmu_seq;
2361
2362 ASSERT(vcpu);
2363 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2364
2365 r = mmu_topup_memory_caches(vcpu);
2366 if (r)
2367 return r;
2368
2369 level = mapping_level(vcpu, gfn);
2370
2371 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
2372
2373 mmu_seq = vcpu->kvm->mmu_notifier_seq;
2374 smp_rmb();
2375 pfn = gfn_to_pfn(vcpu->kvm, gfn);
2376 if (is_error_pfn(pfn))
2377 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
2378 spin_lock(&vcpu->kvm->mmu_lock);
2379 if (mmu_notifier_retry(vcpu, mmu_seq))
2380 goto out_unlock;
2381 kvm_mmu_free_some_pages(vcpu);
2382 r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
2383 level, gfn, pfn);
2384 spin_unlock(&vcpu->kvm->mmu_lock);
2385
2386 return r;
2387
2388out_unlock:
2389 spin_unlock(&vcpu->kvm->mmu_lock);
2390 kvm_release_pfn_clean(pfn);
2391 return 0;
2392}
2393
2394static void nonpaging_free(struct kvm_vcpu *vcpu)
2395{
2396 mmu_free_roots(vcpu);
2397}
2398
2399static int nonpaging_init_context(struct kvm_vcpu *vcpu)
2400{
2401 struct kvm_mmu *context = &vcpu->arch.mmu;
2402
2403 context->new_cr3 = nonpaging_new_cr3;
2404 context->page_fault = nonpaging_page_fault;
2405 context->gva_to_gpa = nonpaging_gva_to_gpa;
2406 context->free = nonpaging_free;
2407 context->prefetch_page = nonpaging_prefetch_page;
2408 context->sync_page = nonpaging_sync_page;
2409 context->invlpg = nonpaging_invlpg;
2410 context->root_level = 0;
2411 context->shadow_root_level = PT32E_ROOT_LEVEL;
2412 context->root_hpa = INVALID_PAGE;
2413 return 0;
2414}
2415
2416void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2417{
2418 ++vcpu->stat.tlb_flush;
2419 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2420}
2421
2422static void paging_new_cr3(struct kvm_vcpu *vcpu)
2423{
2424 pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
2425 mmu_free_roots(vcpu);
2426}
2427
2428static void inject_page_fault(struct kvm_vcpu *vcpu,
2429 u64 addr,
2430 u32 err_code)
2431{
2432 kvm_inject_page_fault(vcpu, addr, err_code);
2433}
2434
2435static void paging_free(struct kvm_vcpu *vcpu)
2436{
2437 nonpaging_free(vcpu);
2438}
2439
2440static bool is_rsvd_bits_set(struct kvm_vcpu *vcpu, u64 gpte, int level)
2441{
2442 int bit7;
2443
2444 bit7 = (gpte >> 7) & 1;
2445 return (gpte & vcpu->arch.mmu.rsvd_bits_mask[bit7][level-1]) != 0;
2446}
2447
2448#define PTTYPE 64
2449#include "paging_tmpl.h"
2450#undef PTTYPE
2451
2452#define PTTYPE 32
2453#include "paging_tmpl.h"
2454#undef PTTYPE
2455
2456static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, int level)
2457{
2458 struct kvm_mmu *context = &vcpu->arch.mmu;
2459 int maxphyaddr = cpuid_maxphyaddr(vcpu);
2460 u64 exb_bit_rsvd = 0;
2461
2462 if (!is_nx(vcpu))
2463 exb_bit_rsvd = rsvd_bits(63, 63);
2464 switch (level) {
2465 case PT32_ROOT_LEVEL:
2466 /* no rsvd bits for 2 level 4K page table entries */
2467 context->rsvd_bits_mask[0][1] = 0;
2468 context->rsvd_bits_mask[0][0] = 0;
2469 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2470
2471 if (!is_pse(vcpu)) {
2472 context->rsvd_bits_mask[1][1] = 0;
2473 break;
2474 }
2475
2476 if (is_cpuid_PSE36())
2477 /* 36bits PSE 4MB page */
2478 context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
2479 else
2480 /* 32 bits PSE 4MB page */
2481 context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
2482 break;
2483 case PT32E_ROOT_LEVEL:
2484 context->rsvd_bits_mask[0][2] =
2485 rsvd_bits(maxphyaddr, 63) |
2486 rsvd_bits(7, 8) | rsvd_bits(1, 2); /* PDPTE */
2487 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2488 rsvd_bits(maxphyaddr, 62); /* PDE */
2489 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2490 rsvd_bits(maxphyaddr, 62); /* PTE */
2491 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2492 rsvd_bits(maxphyaddr, 62) |
2493 rsvd_bits(13, 20); /* large page */
2494 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2495 break;
2496 case PT64_ROOT_LEVEL:
2497 context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
2498 rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2499 context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
2500 rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2501 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2502 rsvd_bits(maxphyaddr, 51);
2503 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2504 rsvd_bits(maxphyaddr, 51);
2505 context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
2506 context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
2507 rsvd_bits(maxphyaddr, 51) |
2508 rsvd_bits(13, 29);
2509 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2510 rsvd_bits(maxphyaddr, 51) |
2511 rsvd_bits(13, 20); /* large page */
2512 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2513 break;
2514 }
2515}
2516
2517static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
2518{
2519 struct kvm_mmu *context = &vcpu->arch.mmu;
2520
2521 ASSERT(is_pae(vcpu));
2522 context->new_cr3 = paging_new_cr3;
2523 context->page_fault = paging64_page_fault;
2524 context->gva_to_gpa = paging64_gva_to_gpa;
2525 context->prefetch_page = paging64_prefetch_page;
2526 context->sync_page = paging64_sync_page;
2527 context->invlpg = paging64_invlpg;
2528 context->free = paging_free;
2529 context->root_level = level;
2530 context->shadow_root_level = level;
2531 context->root_hpa = INVALID_PAGE;
2532 return 0;
2533}
2534
2535static int paging64_init_context(struct kvm_vcpu *vcpu)
2536{
2537 reset_rsvds_bits_mask(vcpu, PT64_ROOT_LEVEL);
2538 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
2539}
2540
2541static int paging32_init_context(struct kvm_vcpu *vcpu)
2542{
2543 struct kvm_mmu *context = &vcpu->arch.mmu;
2544
2545 reset_rsvds_bits_mask(vcpu, PT32_ROOT_LEVEL);
2546 context->new_cr3 = paging_new_cr3;
2547 context->page_fault = paging32_page_fault;
2548 context->gva_to_gpa = paging32_gva_to_gpa;
2549 context->free = paging_free;
2550 context->prefetch_page = paging32_prefetch_page;
2551 context->sync_page = paging32_sync_page;
2552 context->invlpg = paging32_invlpg;
2553 context->root_level = PT32_ROOT_LEVEL;
2554 context->shadow_root_level = PT32E_ROOT_LEVEL;
2555 context->root_hpa = INVALID_PAGE;
2556 return 0;
2557}
2558
2559static int paging32E_init_context(struct kvm_vcpu *vcpu)
2560{
2561 reset_rsvds_bits_mask(vcpu, PT32E_ROOT_LEVEL);
2562 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
2563}
2564
2565static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
2566{
2567 struct kvm_mmu *context = &vcpu->arch.mmu;
2568
2569 context->new_cr3 = nonpaging_new_cr3;
2570 context->page_fault = tdp_page_fault;
2571 context->free = nonpaging_free;
2572 context->prefetch_page = nonpaging_prefetch_page;
2573 context->sync_page = nonpaging_sync_page;
2574 context->invlpg = nonpaging_invlpg;
2575 context->shadow_root_level = kvm_x86_ops->get_tdp_level();
2576 context->root_hpa = INVALID_PAGE;
2577
2578 if (!is_paging(vcpu)) {
2579 context->gva_to_gpa = nonpaging_gva_to_gpa;
2580 context->root_level = 0;
2581 } else if (is_long_mode(vcpu)) {
2582 reset_rsvds_bits_mask(vcpu, PT64_ROOT_LEVEL);
2583 context->gva_to_gpa = paging64_gva_to_gpa;
2584 context->root_level = PT64_ROOT_LEVEL;
2585 } else if (is_pae(vcpu)) {
2586 reset_rsvds_bits_mask(vcpu, PT32E_ROOT_LEVEL);
2587 context->gva_to_gpa = paging64_gva_to_gpa;
2588 context->root_level = PT32E_ROOT_LEVEL;
2589 } else {
2590 reset_rsvds_bits_mask(vcpu, PT32_ROOT_LEVEL);
2591 context->gva_to_gpa = paging32_gva_to_gpa;
2592 context->root_level = PT32_ROOT_LEVEL;
2593 }
2594
2595 return 0;
2596}
2597
2598static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
2599{
2600 int r;
2601
2602 ASSERT(vcpu);
2603 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2604
2605 if (!is_paging(vcpu))
2606 r = nonpaging_init_context(vcpu);
2607 else if (is_long_mode(vcpu))
2608 r = paging64_init_context(vcpu);
2609 else if (is_pae(vcpu))
2610 r = paging32E_init_context(vcpu);
2611 else
2612 r = paging32_init_context(vcpu);
2613
2614 vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
2615 vcpu->arch.mmu.base_role.cr0_wp = is_write_protection(vcpu);
2616
2617 return r;
2618}
2619
2620static int init_kvm_mmu(struct kvm_vcpu *vcpu)
2621{
2622 vcpu->arch.update_pte.pfn = bad_pfn;
2623
2624 if (tdp_enabled)
2625 return init_kvm_tdp_mmu(vcpu);
2626 else
2627 return init_kvm_softmmu(vcpu);
2628}
2629
2630static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
2631{
2632 ASSERT(vcpu);
2633 if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
2634 /* mmu.free() should set root_hpa = INVALID_PAGE */
2635 vcpu->arch.mmu.free(vcpu);
2636}
2637
2638int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
2639{
2640 destroy_kvm_mmu(vcpu);
2641 return init_kvm_mmu(vcpu);
2642}
2643EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
2644
2645int kvm_mmu_load(struct kvm_vcpu *vcpu)
2646{
2647 int r;
2648
2649 r = mmu_topup_memory_caches(vcpu);
2650 if (r)
2651 goto out;
2652 r = mmu_alloc_roots(vcpu);
2653 spin_lock(&vcpu->kvm->mmu_lock);
2654 mmu_sync_roots(vcpu);
2655 spin_unlock(&vcpu->kvm->mmu_lock);
2656 if (r)
2657 goto out;
2658 /* set_cr3() should ensure TLB has been flushed */
2659 kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
2660out:
2661 return r;
2662}
2663EXPORT_SYMBOL_GPL(kvm_mmu_load);
2664
2665void kvm_mmu_unload(struct kvm_vcpu *vcpu)
2666{
2667 mmu_free_roots(vcpu);
2668}
2669
2670static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
2671 struct kvm_mmu_page *sp,
2672 u64 *spte)
2673{
2674 u64 pte;
2675 struct kvm_mmu_page *child;
2676
2677 pte = *spte;
2678 if (is_shadow_present_pte(pte)) {
2679 if (is_last_spte(pte, sp->role.level))
2680 drop_spte(vcpu->kvm, spte, shadow_trap_nonpresent_pte);
2681 else {
2682 child = page_header(pte & PT64_BASE_ADDR_MASK);
2683 mmu_page_remove_parent_pte(child, spte);
2684 }
2685 }
2686 __set_spte(spte, shadow_trap_nonpresent_pte);
2687 if (is_large_pte(pte))
2688 --vcpu->kvm->stat.lpages;
2689}
2690
2691static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
2692 struct kvm_mmu_page *sp,
2693 u64 *spte,
2694 const void *new)
2695{
2696 if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
2697 ++vcpu->kvm->stat.mmu_pde_zapped;
2698 return;
2699 }
2700
2701 if (is_rsvd_bits_set(vcpu, *(u64 *)new, PT_PAGE_TABLE_LEVEL))
2702 return;
2703
2704 ++vcpu->kvm->stat.mmu_pte_updated;
2705 if (!sp->role.cr4_pae)
2706 paging32_update_pte(vcpu, sp, spte, new);
2707 else
2708 paging64_update_pte(vcpu, sp, spte, new);
2709}
2710
2711static bool need_remote_flush(u64 old, u64 new)
2712{
2713 if (!is_shadow_present_pte(old))
2714 return false;
2715 if (!is_shadow_present_pte(new))
2716 return true;
2717 if ((old ^ new) & PT64_BASE_ADDR_MASK)
2718 return true;
2719 old ^= PT64_NX_MASK;
2720 new ^= PT64_NX_MASK;
2721 return (old & ~new & PT64_PERM_MASK) != 0;
2722}
2723
2724static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
2725 bool remote_flush, bool local_flush)
2726{
2727 if (zap_page)
2728 return;
2729
2730 if (remote_flush)
2731 kvm_flush_remote_tlbs(vcpu->kvm);
2732 else if (local_flush)
2733 kvm_mmu_flush_tlb(vcpu);
2734}
2735
2736static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
2737{
2738 u64 *spte = vcpu->arch.last_pte_updated;
2739
2740 return !!(spte && (*spte & shadow_accessed_mask));
2741}
2742
2743static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2744 u64 gpte)
2745{
2746 gfn_t gfn;
2747 pfn_t pfn;
2748
2749 if (!is_present_gpte(gpte))
2750 return;
2751 gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
2752
2753 vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
2754 smp_rmb();
2755 pfn = gfn_to_pfn(vcpu->kvm, gfn);
2756
2757 if (is_error_pfn(pfn)) {
2758 kvm_release_pfn_clean(pfn);
2759 return;
2760 }
2761 vcpu->arch.update_pte.gfn = gfn;
2762 vcpu->arch.update_pte.pfn = pfn;
2763}
2764
2765static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
2766{
2767 u64 *spte = vcpu->arch.last_pte_updated;
2768
2769 if (spte
2770 && vcpu->arch.last_pte_gfn == gfn
2771 && shadow_accessed_mask
2772 && !(*spte & shadow_accessed_mask)
2773 && is_shadow_present_pte(*spte))
2774 set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
2775}
2776
2777void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2778 const u8 *new, int bytes,
2779 bool guest_initiated)
2780{
2781 gfn_t gfn = gpa >> PAGE_SHIFT;
2782 union kvm_mmu_page_role mask = { .word = 0 };
2783 struct kvm_mmu_page *sp;
2784 struct hlist_node *node;
2785 LIST_HEAD(invalid_list);
2786 u64 entry, gentry;
2787 u64 *spte;
2788 unsigned offset = offset_in_page(gpa);
2789 unsigned pte_size;
2790 unsigned page_offset;
2791 unsigned misaligned;
2792 unsigned quadrant;
2793 int level;
2794 int flooded = 0;
2795 int npte;
2796 int r;
2797 int invlpg_counter;
2798 bool remote_flush, local_flush, zap_page;
2799
2800 zap_page = remote_flush = local_flush = false;
2801
2802 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
2803
2804 invlpg_counter = atomic_read(&vcpu->kvm->arch.invlpg_counter);
2805
2806 /*
2807 * Assume that the pte write on a page table of the same type
2808 * as the current vcpu paging mode. This is nearly always true
2809 * (might be false while changing modes). Note it is verified later
2810 * by update_pte().
2811 */
2812 if ((is_pae(vcpu) && bytes == 4) || !new) {
2813 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
2814 if (is_pae(vcpu)) {
2815 gpa &= ~(gpa_t)7;
2816 bytes = 8;
2817 }
2818 r = kvm_read_guest(vcpu->kvm, gpa, &gentry, min(bytes, 8));
2819 if (r)
2820 gentry = 0;
2821 new = (const u8 *)&gentry;
2822 }
2823
2824 switch (bytes) {
2825 case 4:
2826 gentry = *(const u32 *)new;
2827 break;
2828 case 8:
2829 gentry = *(const u64 *)new;
2830 break;
2831 default:
2832 gentry = 0;
2833 break;
2834 }
2835
2836 mmu_guess_page_from_pte_write(vcpu, gpa, gentry);
2837 spin_lock(&vcpu->kvm->mmu_lock);
2838 if (atomic_read(&vcpu->kvm->arch.invlpg_counter) != invlpg_counter)
2839 gentry = 0;
2840 kvm_mmu_access_page(vcpu, gfn);
2841 kvm_mmu_free_some_pages(vcpu);
2842 ++vcpu->kvm->stat.mmu_pte_write;
2843 kvm_mmu_audit(vcpu, "pre pte write");
2844 if (guest_initiated) {
2845 if (gfn == vcpu->arch.last_pt_write_gfn
2846 && !last_updated_pte_accessed(vcpu)) {
2847 ++vcpu->arch.last_pt_write_count;
2848 if (vcpu->arch.last_pt_write_count >= 3)
2849 flooded = 1;
2850 } else {
2851 vcpu->arch.last_pt_write_gfn = gfn;
2852 vcpu->arch.last_pt_write_count = 1;
2853 vcpu->arch.last_pte_updated = NULL;
2854 }
2855 }
2856
2857 mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
2858 for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
2859 pte_size = sp->role.cr4_pae ? 8 : 4;
2860 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
2861 misaligned |= bytes < 4;
2862 if (misaligned || flooded) {
2863 /*
2864 * Misaligned accesses are too much trouble to fix
2865 * up; also, they usually indicate a page is not used
2866 * as a page table.
2867 *
2868 * If we're seeing too many writes to a page,
2869 * it may no longer be a page table, or we may be
2870 * forking, in which case it is better to unmap the
2871 * page.
2872 */
2873 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
2874 gpa, bytes, sp->role.word);
2875 zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2876 &invalid_list);
2877 ++vcpu->kvm->stat.mmu_flooded;
2878 continue;
2879 }
2880 page_offset = offset;
2881 level = sp->role.level;
2882 npte = 1;
2883 if (!sp->role.cr4_pae) {
2884 page_offset <<= 1; /* 32->64 */
2885 /*
2886 * A 32-bit pde maps 4MB while the shadow pdes map
2887 * only 2MB. So we need to double the offset again
2888 * and zap two pdes instead of one.
2889 */
2890 if (level == PT32_ROOT_LEVEL) {
2891 page_offset &= ~7; /* kill rounding error */
2892 page_offset <<= 1;
2893 npte = 2;
2894 }
2895 quadrant = page_offset >> PAGE_SHIFT;
2896 page_offset &= ~PAGE_MASK;
2897 if (quadrant != sp->role.quadrant)
2898 continue;
2899 }
2900 local_flush = true;
2901 spte = &sp->spt[page_offset / sizeof(*spte)];
2902 while (npte--) {
2903 entry = *spte;
2904 mmu_pte_write_zap_pte(vcpu, sp, spte);
2905 if (gentry &&
2906 !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
2907 & mask.word))
2908 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
2909 if (!remote_flush && need_remote_flush(entry, *spte))
2910 remote_flush = true;
2911 ++spte;
2912 }
2913 }
2914 mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
2915 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2916 kvm_mmu_audit(vcpu, "post pte write");
2917 spin_unlock(&vcpu->kvm->mmu_lock);
2918 if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
2919 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
2920 vcpu->arch.update_pte.pfn = bad_pfn;
2921 }
2922}
2923
2924int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2925{
2926 gpa_t gpa;
2927 int r;
2928
2929 if (tdp_enabled)
2930 return 0;
2931
2932 gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
2933
2934 spin_lock(&vcpu->kvm->mmu_lock);
2935 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2936 spin_unlock(&vcpu->kvm->mmu_lock);
2937 return r;
2938}
2939EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
2940
2941void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
2942{
2943 int free_pages;
2944 LIST_HEAD(invalid_list);
2945
2946 free_pages = vcpu->kvm->arch.n_free_mmu_pages;
2947 while (free_pages < KVM_REFILL_PAGES &&
2948 !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
2949 struct kvm_mmu_page *sp;
2950
2951 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
2952 struct kvm_mmu_page, link);
2953 free_pages += kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2954 &invalid_list);
2955 ++vcpu->kvm->stat.mmu_recycled;
2956 }
2957 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2958}
2959
2960int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
2961{
2962 int r;
2963 enum emulation_result er;
2964
2965 r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
2966 if (r < 0)
2967 goto out;
2968
2969 if (!r) {
2970 r = 1;
2971 goto out;
2972 }
2973
2974 r = mmu_topup_memory_caches(vcpu);
2975 if (r)
2976 goto out;
2977
2978 er = emulate_instruction(vcpu, cr2, error_code, 0);
2979
2980 switch (er) {
2981 case EMULATE_DONE:
2982 return 1;
2983 case EMULATE_DO_MMIO:
2984 ++vcpu->stat.mmio_exits;
2985 /* fall through */
2986 case EMULATE_FAIL:
2987 return 0;
2988 default:
2989 BUG();
2990 }
2991out:
2992 return r;
2993}
2994EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
2995
2996void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
2997{
2998 vcpu->arch.mmu.invlpg(vcpu, gva);
2999 kvm_mmu_flush_tlb(vcpu);
3000 ++vcpu->stat.invlpg;
3001}
3002EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
3003
3004void kvm_enable_tdp(void)
3005{
3006 tdp_enabled = true;
3007}
3008EXPORT_SYMBOL_GPL(kvm_enable_tdp);
3009
3010void kvm_disable_tdp(void)
3011{
3012 tdp_enabled = false;
3013}
3014EXPORT_SYMBOL_GPL(kvm_disable_tdp);
3015
3016static void free_mmu_pages(struct kvm_vcpu *vcpu)
3017{
3018 free_page((unsigned long)vcpu->arch.mmu.pae_root);
3019}
3020
3021static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
3022{
3023 struct page *page;
3024 int i;
3025
3026 ASSERT(vcpu);
3027
3028 /*
3029 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
3030 * Therefore we need to allocate shadow page tables in the first
3031 * 4GB of memory, which happens to fit the DMA32 zone.
3032 */
3033 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
3034 if (!page)
3035 return -ENOMEM;
3036
3037 vcpu->arch.mmu.pae_root = page_address(page);
3038 for (i = 0; i < 4; ++i)
3039 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3040
3041 return 0;
3042}
3043
3044int kvm_mmu_create(struct kvm_vcpu *vcpu)
3045{
3046 ASSERT(vcpu);
3047 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3048
3049 return alloc_mmu_pages(vcpu);
3050}
3051
3052int kvm_mmu_setup(struct kvm_vcpu *vcpu)
3053{
3054 ASSERT(vcpu);
3055 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3056
3057 return init_kvm_mmu(vcpu);
3058}
3059
3060void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
3061{
3062 ASSERT(vcpu);
3063
3064 destroy_kvm_mmu(vcpu);
3065 free_mmu_pages(vcpu);
3066 mmu_free_memory_caches(vcpu);
3067}
3068
3069void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
3070{
3071 struct kvm_mmu_page *sp;
3072
3073 list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
3074 int i;
3075 u64 *pt;
3076
3077 if (!test_bit(slot, sp->slot_bitmap))
3078 continue;
3079
3080 pt = sp->spt;
3081 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
3082 /* avoid RMW */
3083 if (is_writable_pte(pt[i]))
3084 pt[i] &= ~PT_WRITABLE_MASK;
3085 }
3086 kvm_flush_remote_tlbs(kvm);
3087}
3088
3089void kvm_mmu_zap_all(struct kvm *kvm)
3090{
3091 struct kvm_mmu_page *sp, *node;
3092 LIST_HEAD(invalid_list);
3093
3094 spin_lock(&kvm->mmu_lock);
3095restart:
3096 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
3097 if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
3098 goto restart;
3099
3100 kvm_mmu_commit_zap_page(kvm, &invalid_list);
3101 spin_unlock(&kvm->mmu_lock);
3102}
3103
3104static int kvm_mmu_remove_some_alloc_mmu_pages(struct kvm *kvm,
3105 struct list_head *invalid_list)
3106{
3107 struct kvm_mmu_page *page;
3108
3109 page = container_of(kvm->arch.active_mmu_pages.prev,
3110 struct kvm_mmu_page, link);
3111 return kvm_mmu_prepare_zap_page(kvm, page, invalid_list);
3112}
3113
3114static int mmu_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
3115{
3116 struct kvm *kvm;
3117 struct kvm *kvm_freed = NULL;
3118 int cache_count = 0;
3119
3120 spin_lock(&kvm_lock);
3121
3122 list_for_each_entry(kvm, &vm_list, vm_list) {
3123 int npages, idx, freed_pages;
3124 LIST_HEAD(invalid_list);
3125
3126 idx = srcu_read_lock(&kvm->srcu);
3127 spin_lock(&kvm->mmu_lock);
3128 npages = kvm->arch.n_alloc_mmu_pages -
3129 kvm->arch.n_free_mmu_pages;
3130 cache_count += npages;
3131 if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
3132 freed_pages = kvm_mmu_remove_some_alloc_mmu_pages(kvm,
3133 &invalid_list);
3134 cache_count -= freed_pages;
3135 kvm_freed = kvm;
3136 }
3137 nr_to_scan--;
3138
3139 kvm_mmu_commit_zap_page(kvm, &invalid_list);
3140 spin_unlock(&kvm->mmu_lock);
3141 srcu_read_unlock(&kvm->srcu, idx);
3142 }
3143 if (kvm_freed)
3144 list_move_tail(&kvm_freed->vm_list, &vm_list);
3145
3146 spin_unlock(&kvm_lock);
3147
3148 return cache_count;
3149}
3150
3151static struct shrinker mmu_shrinker = {
3152 .shrink = mmu_shrink,
3153 .seeks = DEFAULT_SEEKS * 10,
3154};
3155
3156static void mmu_destroy_caches(void)
3157{
3158 if (pte_chain_cache)
3159 kmem_cache_destroy(pte_chain_cache);
3160 if (rmap_desc_cache)
3161 kmem_cache_destroy(rmap_desc_cache);
3162 if (mmu_page_header_cache)
3163 kmem_cache_destroy(mmu_page_header_cache);
3164}
3165
3166void kvm_mmu_module_exit(void)
3167{
3168 mmu_destroy_caches();
3169 unregister_shrinker(&mmu_shrinker);
3170}
3171
3172int kvm_mmu_module_init(void)
3173{
3174 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
3175 sizeof(struct kvm_pte_chain),
3176 0, 0, NULL);
3177 if (!pte_chain_cache)
3178 goto nomem;
3179 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
3180 sizeof(struct kvm_rmap_desc),
3181 0, 0, NULL);
3182 if (!rmap_desc_cache)
3183 goto nomem;
3184
3185 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
3186 sizeof(struct kvm_mmu_page),
3187 0, 0, NULL);
3188 if (!mmu_page_header_cache)
3189 goto nomem;
3190
3191 register_shrinker(&mmu_shrinker);
3192
3193 return 0;
3194
3195nomem:
3196 mmu_destroy_caches();
3197 return -ENOMEM;
3198}
3199
3200/*
3201 * Caculate mmu pages needed for kvm.
3202 */
3203unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
3204{
3205 int i;
3206 unsigned int nr_mmu_pages;
3207 unsigned int nr_pages = 0;
3208 struct kvm_memslots *slots;
3209
3210 slots = kvm_memslots(kvm);
3211
3212 for (i = 0; i < slots->nmemslots; i++)
3213 nr_pages += slots->memslots[i].npages;
3214
3215 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
3216 nr_mmu_pages = max(nr_mmu_pages,
3217 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
3218
3219 return nr_mmu_pages;
3220}
3221
3222static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
3223 unsigned len)
3224{
3225 if (len > buffer->len)
3226 return NULL;
3227 return buffer->ptr;
3228}
3229
3230static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
3231 unsigned len)
3232{
3233 void *ret;
3234
3235 ret = pv_mmu_peek_buffer(buffer, len);
3236 if (!ret)
3237 return ret;
3238 buffer->ptr += len;
3239 buffer->len -= len;
3240 buffer->processed += len;
3241 return ret;
3242}
3243
3244static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
3245 gpa_t addr, gpa_t value)
3246{
3247 int bytes = 8;
3248 int r;
3249
3250 if (!is_long_mode(vcpu) && !is_pae(vcpu))
3251 bytes = 4;
3252
3253 r = mmu_topup_memory_caches(vcpu);
3254 if (r)
3255 return r;
3256
3257 if (!emulator_write_phys(vcpu, addr, &value, bytes))
3258 return -EFAULT;
3259
3260 return 1;
3261}
3262
3263static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
3264{
3265 (void)kvm_set_cr3(vcpu, vcpu->arch.cr3);
3266 return 1;
3267}
3268
3269static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
3270{
3271 spin_lock(&vcpu->kvm->mmu_lock);
3272 mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
3273 spin_unlock(&vcpu->kvm->mmu_lock);
3274 return 1;
3275}
3276
3277static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
3278 struct kvm_pv_mmu_op_buffer *buffer)
3279{
3280 struct kvm_mmu_op_header *header;
3281
3282 header = pv_mmu_peek_buffer(buffer, sizeof *header);
3283 if (!header)
3284 return 0;
3285 switch (header->op) {
3286 case KVM_MMU_OP_WRITE_PTE: {
3287 struct kvm_mmu_op_write_pte *wpte;
3288
3289 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
3290 if (!wpte)
3291 return 0;
3292 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
3293 wpte->pte_val);
3294 }
3295 case KVM_MMU_OP_FLUSH_TLB: {
3296 struct kvm_mmu_op_flush_tlb *ftlb;
3297
3298 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
3299 if (!ftlb)
3300 return 0;
3301 return kvm_pv_mmu_flush_tlb(vcpu);
3302 }
3303 case KVM_MMU_OP_RELEASE_PT: {
3304 struct kvm_mmu_op_release_pt *rpt;
3305
3306 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
3307 if (!rpt)
3308 return 0;
3309 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
3310 }
3311 default: return 0;
3312 }
3313}
3314
3315int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
3316 gpa_t addr, unsigned long *ret)
3317{
3318 int r;
3319 struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
3320
3321 buffer->ptr = buffer->buf;
3322 buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
3323 buffer->processed = 0;
3324
3325 r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
3326 if (r)
3327 goto out;
3328
3329 while (buffer->len) {
3330 r = kvm_pv_mmu_op_one(vcpu, buffer);
3331 if (r < 0)
3332 goto out;
3333 if (r == 0)
3334 break;
3335 }
3336
3337 r = 1;
3338out:
3339 *ret = buffer->processed;
3340 return r;
3341}
3342
3343int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
3344{
3345 struct kvm_shadow_walk_iterator iterator;
3346 int nr_sptes = 0;
3347
3348 spin_lock(&vcpu->kvm->mmu_lock);
3349 for_each_shadow_entry(vcpu, addr, iterator) {
3350 sptes[iterator.level-1] = *iterator.sptep;
3351 nr_sptes++;
3352 if (!is_shadow_present_pte(*iterator.sptep))
3353 break;
3354 }
3355 spin_unlock(&vcpu->kvm->mmu_lock);
3356
3357 return nr_sptes;
3358}
3359EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
3360
3361#ifdef AUDIT
3362
3363static const char *audit_msg;
3364
3365static gva_t canonicalize(gva_t gva)
3366{
3367#ifdef CONFIG_X86_64
3368 gva = (long long)(gva << 16) >> 16;
3369#endif
3370 return gva;
3371}
3372
3373
3374typedef void (*inspect_spte_fn) (struct kvm *kvm, u64 *sptep);
3375
3376static void __mmu_spte_walk(struct kvm *kvm, struct kvm_mmu_page *sp,
3377 inspect_spte_fn fn)
3378{
3379 int i;
3380
3381 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
3382 u64 ent = sp->spt[i];
3383
3384 if (is_shadow_present_pte(ent)) {
3385 if (!is_last_spte(ent, sp->role.level)) {
3386 struct kvm_mmu_page *child;
3387 child = page_header(ent & PT64_BASE_ADDR_MASK);
3388 __mmu_spte_walk(kvm, child, fn);
3389 } else
3390 fn(kvm, &sp->spt[i]);
3391 }
3392 }
3393}
3394
3395static void mmu_spte_walk(struct kvm_vcpu *vcpu, inspect_spte_fn fn)
3396{
3397 int i;
3398 struct kvm_mmu_page *sp;
3399
3400 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3401 return;
3402 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3403 hpa_t root = vcpu->arch.mmu.root_hpa;
3404 sp = page_header(root);
3405 __mmu_spte_walk(vcpu->kvm, sp, fn);
3406 return;
3407 }
3408 for (i = 0; i < 4; ++i) {
3409 hpa_t root = vcpu->arch.mmu.pae_root[i];
3410
3411 if (root && VALID_PAGE(root)) {
3412 root &= PT64_BASE_ADDR_MASK;
3413 sp = page_header(root);
3414 __mmu_spte_walk(vcpu->kvm, sp, fn);
3415 }
3416 }
3417 return;
3418}
3419
3420static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
3421 gva_t va, int level)
3422{
3423 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
3424 int i;
3425 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
3426
3427 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
3428 u64 ent = pt[i];
3429
3430 if (ent == shadow_trap_nonpresent_pte)
3431 continue;
3432
3433 va = canonicalize(va);
3434 if (is_shadow_present_pte(ent) && !is_last_spte(ent, level))
3435 audit_mappings_page(vcpu, ent, va, level - 1);
3436 else {
3437 gpa_t gpa = kvm_mmu_gva_to_gpa_read(vcpu, va, NULL);
3438 gfn_t gfn = gpa >> PAGE_SHIFT;
3439 pfn_t pfn = gfn_to_pfn(vcpu->kvm, gfn);
3440 hpa_t hpa = (hpa_t)pfn << PAGE_SHIFT;
3441
3442 if (is_error_pfn(pfn)) {
3443 kvm_release_pfn_clean(pfn);
3444 continue;
3445 }
3446
3447 if (is_shadow_present_pte(ent)
3448 && (ent & PT64_BASE_ADDR_MASK) != hpa)
3449 printk(KERN_ERR "xx audit error: (%s) levels %d"
3450 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
3451 audit_msg, vcpu->arch.mmu.root_level,
3452 va, gpa, hpa, ent,
3453 is_shadow_present_pte(ent));
3454 else if (ent == shadow_notrap_nonpresent_pte
3455 && !is_error_hpa(hpa))
3456 printk(KERN_ERR "audit: (%s) notrap shadow,"
3457 " valid guest gva %lx\n", audit_msg, va);
3458 kvm_release_pfn_clean(pfn);
3459
3460 }
3461 }
3462}
3463
3464static void audit_mappings(struct kvm_vcpu *vcpu)
3465{
3466 unsigned i;
3467
3468 if (vcpu->arch.mmu.root_level == 4)
3469 audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
3470 else
3471 for (i = 0; i < 4; ++i)
3472 if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
3473 audit_mappings_page(vcpu,
3474 vcpu->arch.mmu.pae_root[i],
3475 i << 30,
3476 2);
3477}
3478
3479static int count_rmaps(struct kvm_vcpu *vcpu)
3480{
3481 struct kvm *kvm = vcpu->kvm;
3482 struct kvm_memslots *slots;
3483 int nmaps = 0;
3484 int i, j, k, idx;
3485
3486 idx = srcu_read_lock(&kvm->srcu);
3487 slots = kvm_memslots(kvm);
3488 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
3489 struct kvm_memory_slot *m = &slots->memslots[i];
3490 struct kvm_rmap_desc *d;
3491
3492 for (j = 0; j < m->npages; ++j) {
3493 unsigned long *rmapp = &m->rmap[j];
3494
3495 if (!*rmapp)
3496 continue;
3497 if (!(*rmapp & 1)) {
3498 ++nmaps;
3499 continue;
3500 }
3501 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
3502 while (d) {
3503 for (k = 0; k < RMAP_EXT; ++k)
3504 if (d->sptes[k])
3505 ++nmaps;
3506 else
3507 break;
3508 d = d->more;
3509 }
3510 }
3511 }
3512 srcu_read_unlock(&kvm->srcu, idx);
3513 return nmaps;
3514}
3515
3516void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
3517{
3518 unsigned long *rmapp;
3519 struct kvm_mmu_page *rev_sp;
3520 gfn_t gfn;
3521
3522 if (is_writable_pte(*sptep)) {
3523 rev_sp = page_header(__pa(sptep));
3524 gfn = kvm_mmu_page_get_gfn(rev_sp, sptep - rev_sp->spt);
3525
3526 if (!gfn_to_memslot(kvm, gfn)) {
3527 if (!printk_ratelimit())
3528 return;
3529 printk(KERN_ERR "%s: no memslot for gfn %ld\n",
3530 audit_msg, gfn);
3531 printk(KERN_ERR "%s: index %ld of sp (gfn=%lx)\n",
3532 audit_msg, (long int)(sptep - rev_sp->spt),
3533 rev_sp->gfn);
3534 dump_stack();
3535 return;
3536 }
3537
3538 rmapp = gfn_to_rmap(kvm, gfn, rev_sp->role.level);
3539 if (!*rmapp) {
3540 if (!printk_ratelimit())
3541 return;
3542 printk(KERN_ERR "%s: no rmap for writable spte %llx\n",
3543 audit_msg, *sptep);
3544 dump_stack();
3545 }
3546 }
3547
3548}
3549
3550void audit_writable_sptes_have_rmaps(struct kvm_vcpu *vcpu)
3551{
3552 mmu_spte_walk(vcpu, inspect_spte_has_rmap);
3553}
3554
3555static void check_writable_mappings_rmap(struct kvm_vcpu *vcpu)
3556{
3557 struct kvm_mmu_page *sp;
3558 int i;
3559
3560 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
3561 u64 *pt = sp->spt;
3562
3563 if (sp->role.level != PT_PAGE_TABLE_LEVEL)
3564 continue;
3565
3566 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
3567 u64 ent = pt[i];
3568
3569 if (!(ent & PT_PRESENT_MASK))
3570 continue;
3571 if (!is_writable_pte(ent))
3572 continue;
3573 inspect_spte_has_rmap(vcpu->kvm, &pt[i]);
3574 }
3575 }
3576 return;
3577}
3578
3579static void audit_rmap(struct kvm_vcpu *vcpu)
3580{
3581 check_writable_mappings_rmap(vcpu);
3582 count_rmaps(vcpu);
3583}
3584
3585static void audit_write_protection(struct kvm_vcpu *vcpu)
3586{
3587 struct kvm_mmu_page *sp;
3588 struct kvm_memory_slot *slot;
3589 unsigned long *rmapp;
3590 u64 *spte;
3591 gfn_t gfn;
3592
3593 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
3594 if (sp->role.direct)
3595 continue;
3596 if (sp->unsync)
3597 continue;
3598
3599 slot = gfn_to_memslot(vcpu->kvm, sp->gfn);
3600 rmapp = &slot->rmap[gfn - slot->base_gfn];
3601
3602 spte = rmap_next(vcpu->kvm, rmapp, NULL);
3603 while (spte) {
3604 if (is_writable_pte(*spte))
3605 printk(KERN_ERR "%s: (%s) shadow page has "
3606 "writable mappings: gfn %lx role %x\n",
3607 __func__, audit_msg, sp->gfn,
3608 sp->role.word);
3609 spte = rmap_next(vcpu->kvm, rmapp, spte);
3610 }
3611 }
3612}
3613
3614static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
3615{
3616 int olddbg = dbg;
3617
3618 dbg = 0;
3619 audit_msg = msg;
3620 audit_rmap(vcpu);
3621 audit_write_protection(vcpu);
3622 if (strcmp("pre pte write", audit_msg) != 0)
3623 audit_mappings(vcpu);
3624 audit_writable_sptes_have_rmaps(vcpu);
3625 dbg = olddbg;
3626}
3627
3628#endif
Linux kernel - Deliverables
This page took 0.035045 seconds and 5 git commands to generate.