4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
45 #include <linux/vmalloc.h>
46 #include <linux/module.h>
47 #include <linux/gfp.h>
49 #include <asm/pgtable.h>
50 #include <asm/tlbflush.h>
51 #include <asm/fixmap.h>
52 #include <asm/mmu_context.h>
53 #include <asm/setup.h>
54 #include <asm/paravirt.h>
56 #include <asm/linkage.h>
59 #include <asm/xen/hypercall.h>
60 #include <asm/xen/hypervisor.h>
64 #include <xen/interface/xen.h>
65 #include <xen/interface/hvm/hvm_op.h>
66 #include <xen/interface/version.h>
67 #include <xen/interface/memory.h>
68 #include <xen/hvc-console.h>
70 #include "multicalls.h"
74 #define MMU_UPDATE_HISTO 30
77 * Protects atomic reservation decrease/increase against concurrent increases.
78 * Also protects non-atomic updates of current_pages and driver_pages, and
81 DEFINE_SPINLOCK(xen_reservation_lock
);
83 #ifdef CONFIG_XEN_DEBUG_FS
87 u32 pgd_update_pinned
;
88 u32 pgd_update_batched
;
91 u32 pud_update_pinned
;
92 u32 pud_update_batched
;
95 u32 pmd_update_pinned
;
96 u32 pmd_update_batched
;
99 u32 pte_update_pinned
;
100 u32 pte_update_batched
;
103 u32 mmu_update_extended
;
104 u32 mmu_update_histo
[MMU_UPDATE_HISTO
];
107 u32 prot_commit_batched
;
110 u32 set_pte_at_batched
;
111 u32 set_pte_at_pinned
;
112 u32 set_pte_at_current
;
113 u32 set_pte_at_kernel
;
116 static u8 zero_stats
;
118 static inline void check_zero(void)
120 if (unlikely(zero_stats
)) {
121 memset(&mmu_stats
, 0, sizeof(mmu_stats
));
126 #define ADD_STATS(elem, val) \
127 do { check_zero(); mmu_stats.elem += (val); } while(0)
129 #else /* !CONFIG_XEN_DEBUG_FS */
131 #define ADD_STATS(elem, val) do { (void)(val); } while(0)
133 #endif /* CONFIG_XEN_DEBUG_FS */
137 * Identity map, in addition to plain kernel map. This needs to be
138 * large enough to allocate page table pages to allocate the rest.
139 * Each page can map 2MB.
141 static pte_t level1_ident_pgt
[PTRS_PER_PTE
* 4] __page_aligned_bss
;
144 /* l3 pud for userspace vsyscall mapping */
145 static pud_t level3_user_vsyscall
[PTRS_PER_PUD
] __page_aligned_bss
;
146 #endif /* CONFIG_X86_64 */
149 * Note about cr3 (pagetable base) values:
151 * xen_cr3 contains the current logical cr3 value; it contains the
152 * last set cr3. This may not be the current effective cr3, because
153 * its update may be being lazily deferred. However, a vcpu looking
154 * at its own cr3 can use this value knowing that it everything will
155 * be self-consistent.
157 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
158 * hypercall to set the vcpu cr3 is complete (so it may be a little
159 * out of date, but it will never be set early). If one vcpu is
160 * looking at another vcpu's cr3 value, it should use this variable.
162 DEFINE_PER_CPU(unsigned long, xen_cr3
); /* cr3 stored as physaddr */
163 DEFINE_PER_CPU(unsigned long, xen_current_cr3
); /* actual vcpu cr3 */
167 * Just beyond the highest usermode address. STACK_TOP_MAX has a
168 * redzone above it, so round it up to a PGD boundary.
170 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
173 #define P2M_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
174 #define TOP_ENTRIES (MAX_DOMAIN_PAGES / P2M_ENTRIES_PER_PAGE)
176 /* Placeholder for holes in the address space */
177 static unsigned long p2m_missing
[P2M_ENTRIES_PER_PAGE
] __page_aligned_data
=
178 { [ 0 ... P2M_ENTRIES_PER_PAGE
-1 ] = ~0UL };
180 /* Array of pointers to pages containing p2m entries */
181 static unsigned long *p2m_top
[TOP_ENTRIES
] __page_aligned_data
=
182 { [ 0 ... TOP_ENTRIES
- 1] = &p2m_missing
[0] };
184 /* Arrays of p2m arrays expressed in mfns used for save/restore */
185 static unsigned long p2m_top_mfn
[TOP_ENTRIES
] __page_aligned_bss
;
187 static unsigned long p2m_top_mfn_list
[TOP_ENTRIES
/ P2M_ENTRIES_PER_PAGE
]
190 static inline unsigned p2m_top_index(unsigned long pfn
)
192 BUG_ON(pfn
>= MAX_DOMAIN_PAGES
);
193 return pfn
/ P2M_ENTRIES_PER_PAGE
;
196 static inline unsigned p2m_index(unsigned long pfn
)
198 return pfn
% P2M_ENTRIES_PER_PAGE
;
201 /* Build the parallel p2m_top_mfn structures */
202 void xen_build_mfn_list_list(void)
206 for (pfn
= 0; pfn
< MAX_DOMAIN_PAGES
; pfn
+= P2M_ENTRIES_PER_PAGE
) {
207 unsigned topidx
= p2m_top_index(pfn
);
209 p2m_top_mfn
[topidx
] = virt_to_mfn(p2m_top
[topidx
]);
212 for (idx
= 0; idx
< ARRAY_SIZE(p2m_top_mfn_list
); idx
++) {
213 unsigned topidx
= idx
* P2M_ENTRIES_PER_PAGE
;
214 p2m_top_mfn_list
[idx
] = virt_to_mfn(&p2m_top_mfn
[topidx
]);
218 void xen_setup_mfn_list_list(void)
220 BUG_ON(HYPERVISOR_shared_info
== &xen_dummy_shared_info
);
222 HYPERVISOR_shared_info
->arch
.pfn_to_mfn_frame_list_list
=
223 virt_to_mfn(p2m_top_mfn_list
);
224 HYPERVISOR_shared_info
->arch
.max_pfn
= xen_start_info
->nr_pages
;
227 /* Set up p2m_top to point to the domain-builder provided p2m pages */
228 void __init
xen_build_dynamic_phys_to_machine(void)
230 unsigned long *mfn_list
= (unsigned long *)xen_start_info
->mfn_list
;
231 unsigned long max_pfn
= min(MAX_DOMAIN_PAGES
, xen_start_info
->nr_pages
);
234 for (pfn
= 0; pfn
< max_pfn
; pfn
+= P2M_ENTRIES_PER_PAGE
) {
235 unsigned topidx
= p2m_top_index(pfn
);
237 p2m_top
[topidx
] = &mfn_list
[pfn
];
240 xen_build_mfn_list_list();
243 unsigned long get_phys_to_machine(unsigned long pfn
)
245 unsigned topidx
, idx
;
247 if (unlikely(pfn
>= MAX_DOMAIN_PAGES
))
248 return INVALID_P2M_ENTRY
;
250 topidx
= p2m_top_index(pfn
);
251 idx
= p2m_index(pfn
);
252 return p2m_top
[topidx
][idx
];
254 EXPORT_SYMBOL_GPL(get_phys_to_machine
);
256 /* install a new p2m_top page */
257 bool install_p2mtop_page(unsigned long pfn
, unsigned long *p
)
259 unsigned topidx
= p2m_top_index(pfn
);
260 unsigned long **pfnp
, *mfnp
;
263 pfnp
= &p2m_top
[topidx
];
264 mfnp
= &p2m_top_mfn
[topidx
];
266 for (i
= 0; i
< P2M_ENTRIES_PER_PAGE
; i
++)
267 p
[i
] = INVALID_P2M_ENTRY
;
269 if (cmpxchg(pfnp
, p2m_missing
, p
) == p2m_missing
) {
270 *mfnp
= virt_to_mfn(p
);
277 static void alloc_p2m(unsigned long pfn
)
281 p
= (void *)__get_free_page(GFP_KERNEL
| __GFP_NOFAIL
);
284 if (!install_p2mtop_page(pfn
, p
))
285 free_page((unsigned long)p
);
288 /* Try to install p2m mapping; fail if intermediate bits missing */
289 bool __set_phys_to_machine(unsigned long pfn
, unsigned long mfn
)
291 unsigned topidx
, idx
;
293 if (unlikely(pfn
>= MAX_DOMAIN_PAGES
)) {
294 BUG_ON(mfn
!= INVALID_P2M_ENTRY
);
298 topidx
= p2m_top_index(pfn
);
299 if (p2m_top
[topidx
] == p2m_missing
) {
300 if (mfn
== INVALID_P2M_ENTRY
)
305 idx
= p2m_index(pfn
);
306 p2m_top
[topidx
][idx
] = mfn
;
311 void set_phys_to_machine(unsigned long pfn
, unsigned long mfn
)
313 if (unlikely(xen_feature(XENFEAT_auto_translated_physmap
))) {
314 BUG_ON(pfn
!= mfn
&& mfn
!= INVALID_P2M_ENTRY
);
318 if (unlikely(!__set_phys_to_machine(pfn
, mfn
))) {
321 if (!__set_phys_to_machine(pfn
, mfn
))
326 unsigned long arbitrary_virt_to_mfn(void *vaddr
)
328 xmaddr_t maddr
= arbitrary_virt_to_machine(vaddr
);
330 return PFN_DOWN(maddr
.maddr
);
333 xmaddr_t
arbitrary_virt_to_machine(void *vaddr
)
335 unsigned long address
= (unsigned long)vaddr
;
341 * if the PFN is in the linear mapped vaddr range, we can just use
342 * the (quick) virt_to_machine() p2m lookup
344 if (virt_addr_valid(vaddr
))
345 return virt_to_machine(vaddr
);
347 /* otherwise we have to do a (slower) full page-table walk */
349 pte
= lookup_address(address
, &level
);
351 offset
= address
& ~PAGE_MASK
;
352 return XMADDR(((phys_addr_t
)pte_mfn(*pte
) << PAGE_SHIFT
) + offset
);
355 void make_lowmem_page_readonly(void *vaddr
)
358 unsigned long address
= (unsigned long)vaddr
;
361 pte
= lookup_address(address
, &level
);
364 ptev
= pte_wrprotect(*pte
);
366 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
370 void make_lowmem_page_readwrite(void *vaddr
)
373 unsigned long address
= (unsigned long)vaddr
;
376 pte
= lookup_address(address
, &level
);
379 ptev
= pte_mkwrite(*pte
);
381 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
386 static bool xen_page_pinned(void *ptr
)
388 struct page
*page
= virt_to_page(ptr
);
390 return PagePinned(page
);
393 static bool xen_iomap_pte(pte_t pte
)
395 return pte_flags(pte
) & _PAGE_IOMAP
;
398 static void xen_set_iomap_pte(pte_t
*ptep
, pte_t pteval
)
400 struct multicall_space mcs
;
401 struct mmu_update
*u
;
403 mcs
= xen_mc_entry(sizeof(*u
));
406 /* ptep might be kmapped when using 32-bit HIGHPTE */
407 u
->ptr
= arbitrary_virt_to_machine(ptep
).maddr
;
408 u
->val
= pte_val_ma(pteval
);
410 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_IO
);
412 xen_mc_issue(PARAVIRT_LAZY_MMU
);
415 static void xen_extend_mmu_update(const struct mmu_update
*update
)
417 struct multicall_space mcs
;
418 struct mmu_update
*u
;
420 mcs
= xen_mc_extend_args(__HYPERVISOR_mmu_update
, sizeof(*u
));
422 if (mcs
.mc
!= NULL
) {
423 ADD_STATS(mmu_update_extended
, 1);
424 ADD_STATS(mmu_update_histo
[mcs
.mc
->args
[1]], -1);
428 if (mcs
.mc
->args
[1] < MMU_UPDATE_HISTO
)
429 ADD_STATS(mmu_update_histo
[mcs
.mc
->args
[1]], 1);
431 ADD_STATS(mmu_update_histo
[0], 1);
433 ADD_STATS(mmu_update
, 1);
434 mcs
= __xen_mc_entry(sizeof(*u
));
435 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
436 ADD_STATS(mmu_update_histo
[1], 1);
443 void xen_set_pmd_hyper(pmd_t
*ptr
, pmd_t val
)
451 /* ptr may be ioremapped for 64-bit pagetable setup */
452 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
453 u
.val
= pmd_val_ma(val
);
454 xen_extend_mmu_update(&u
);
456 ADD_STATS(pmd_update_batched
, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU
);
458 xen_mc_issue(PARAVIRT_LAZY_MMU
);
463 void xen_set_pmd(pmd_t
*ptr
, pmd_t val
)
465 ADD_STATS(pmd_update
, 1);
467 /* If page is not pinned, we can just update the entry
469 if (!xen_page_pinned(ptr
)) {
474 ADD_STATS(pmd_update_pinned
, 1);
476 xen_set_pmd_hyper(ptr
, val
);
480 * Associate a virtual page frame with a given physical page frame
481 * and protection flags for that frame.
483 void set_pte_mfn(unsigned long vaddr
, unsigned long mfn
, pgprot_t flags
)
485 set_pte_vaddr(vaddr
, mfn_pte(mfn
, flags
));
488 void xen_set_pte_at(struct mm_struct
*mm
, unsigned long addr
,
489 pte_t
*ptep
, pte_t pteval
)
491 if (xen_iomap_pte(pteval
)) {
492 xen_set_iomap_pte(ptep
, pteval
);
496 ADD_STATS(set_pte_at
, 1);
497 // ADD_STATS(set_pte_at_pinned, xen_page_pinned(ptep));
498 ADD_STATS(set_pte_at_current
, mm
== current
->mm
);
499 ADD_STATS(set_pte_at_kernel
, mm
== &init_mm
);
501 if (mm
== current
->mm
|| mm
== &init_mm
) {
502 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU
) {
503 struct multicall_space mcs
;
504 mcs
= xen_mc_entry(0);
506 MULTI_update_va_mapping(mcs
.mc
, addr
, pteval
, 0);
507 ADD_STATS(set_pte_at_batched
, 1);
508 xen_mc_issue(PARAVIRT_LAZY_MMU
);
511 if (HYPERVISOR_update_va_mapping(addr
, pteval
, 0) == 0)
514 xen_set_pte(ptep
, pteval
);
519 pte_t
xen_ptep_modify_prot_start(struct mm_struct
*mm
,
520 unsigned long addr
, pte_t
*ptep
)
522 /* Just return the pte as-is. We preserve the bits on commit */
526 void xen_ptep_modify_prot_commit(struct mm_struct
*mm
, unsigned long addr
,
527 pte_t
*ptep
, pte_t pte
)
533 u
.ptr
= arbitrary_virt_to_machine(ptep
).maddr
| MMU_PT_UPDATE_PRESERVE_AD
;
534 u
.val
= pte_val_ma(pte
);
535 xen_extend_mmu_update(&u
);
537 ADD_STATS(prot_commit
, 1);
538 ADD_STATS(prot_commit_batched
, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU
);
540 xen_mc_issue(PARAVIRT_LAZY_MMU
);
543 /* Assume pteval_t is equivalent to all the other *val_t types. */
544 static pteval_t
pte_mfn_to_pfn(pteval_t val
)
546 if (val
& _PAGE_PRESENT
) {
547 unsigned long mfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
548 pteval_t flags
= val
& PTE_FLAGS_MASK
;
549 val
= ((pteval_t
)mfn_to_pfn(mfn
) << PAGE_SHIFT
) | flags
;
555 static pteval_t
pte_pfn_to_mfn(pteval_t val
)
557 if (val
& _PAGE_PRESENT
) {
558 unsigned long pfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
559 pteval_t flags
= val
& PTE_FLAGS_MASK
;
560 val
= ((pteval_t
)pfn_to_mfn(pfn
) << PAGE_SHIFT
) | flags
;
566 static pteval_t
iomap_pte(pteval_t val
)
568 if (val
& _PAGE_PRESENT
) {
569 unsigned long pfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
570 pteval_t flags
= val
& PTE_FLAGS_MASK
;
572 /* We assume the pte frame number is a MFN, so
573 just use it as-is. */
574 val
= ((pteval_t
)pfn
<< PAGE_SHIFT
) | flags
;
580 pteval_t
xen_pte_val(pte_t pte
)
582 if (xen_initial_domain() && (pte
.pte
& _PAGE_IOMAP
))
585 return pte_mfn_to_pfn(pte
.pte
);
587 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val
);
589 pgdval_t
xen_pgd_val(pgd_t pgd
)
591 return pte_mfn_to_pfn(pgd
.pgd
);
593 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val
);
595 pte_t
xen_make_pte(pteval_t pte
)
597 phys_addr_t addr
= (pte
& PTE_PFN_MASK
);
600 * Unprivileged domains are allowed to do IOMAPpings for
601 * PCI passthrough, but not map ISA space. The ISA
602 * mappings are just dummy local mappings to keep other
603 * parts of the kernel happy.
605 if (unlikely(pte
& _PAGE_IOMAP
) &&
606 (xen_initial_domain() || addr
>= ISA_END_ADDRESS
)) {
607 pte
= iomap_pte(pte
);
610 pte
= pte_pfn_to_mfn(pte
);
613 return native_make_pte(pte
);
615 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte
);
617 pgd_t
xen_make_pgd(pgdval_t pgd
)
619 pgd
= pte_pfn_to_mfn(pgd
);
620 return native_make_pgd(pgd
);
622 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd
);
624 pmdval_t
xen_pmd_val(pmd_t pmd
)
626 return pte_mfn_to_pfn(pmd
.pmd
);
628 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val
);
630 void xen_set_pud_hyper(pud_t
*ptr
, pud_t val
)
638 /* ptr may be ioremapped for 64-bit pagetable setup */
639 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
640 u
.val
= pud_val_ma(val
);
641 xen_extend_mmu_update(&u
);
643 ADD_STATS(pud_update_batched
, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU
);
645 xen_mc_issue(PARAVIRT_LAZY_MMU
);
650 void xen_set_pud(pud_t
*ptr
, pud_t val
)
652 ADD_STATS(pud_update
, 1);
654 /* If page is not pinned, we can just update the entry
656 if (!xen_page_pinned(ptr
)) {
661 ADD_STATS(pud_update_pinned
, 1);
663 xen_set_pud_hyper(ptr
, val
);
666 void xen_set_pte(pte_t
*ptep
, pte_t pte
)
668 if (xen_iomap_pte(pte
)) {
669 xen_set_iomap_pte(ptep
, pte
);
673 ADD_STATS(pte_update
, 1);
674 // ADD_STATS(pte_update_pinned, xen_page_pinned(ptep));
675 ADD_STATS(pte_update_batched
, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU
);
677 #ifdef CONFIG_X86_PAE
678 ptep
->pte_high
= pte
.pte_high
;
680 ptep
->pte_low
= pte
.pte_low
;
686 #ifdef CONFIG_X86_PAE
687 void xen_set_pte_atomic(pte_t
*ptep
, pte_t pte
)
689 if (xen_iomap_pte(pte
)) {
690 xen_set_iomap_pte(ptep
, pte
);
694 set_64bit((u64
*)ptep
, native_pte_val(pte
));
697 void xen_pte_clear(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
700 smp_wmb(); /* make sure low gets written first */
704 void xen_pmd_clear(pmd_t
*pmdp
)
706 set_pmd(pmdp
, __pmd(0));
708 #endif /* CONFIG_X86_PAE */
710 pmd_t
xen_make_pmd(pmdval_t pmd
)
712 pmd
= pte_pfn_to_mfn(pmd
);
713 return native_make_pmd(pmd
);
715 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd
);
717 #if PAGETABLE_LEVELS == 4
718 pudval_t
xen_pud_val(pud_t pud
)
720 return pte_mfn_to_pfn(pud
.pud
);
722 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val
);
724 pud_t
xen_make_pud(pudval_t pud
)
726 pud
= pte_pfn_to_mfn(pud
);
728 return native_make_pud(pud
);
730 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud
);
732 pgd_t
*xen_get_user_pgd(pgd_t
*pgd
)
734 pgd_t
*pgd_page
= (pgd_t
*)(((unsigned long)pgd
) & PAGE_MASK
);
735 unsigned offset
= pgd
- pgd_page
;
736 pgd_t
*user_ptr
= NULL
;
738 if (offset
< pgd_index(USER_LIMIT
)) {
739 struct page
*page
= virt_to_page(pgd_page
);
740 user_ptr
= (pgd_t
*)page
->private;
748 static void __xen_set_pgd_hyper(pgd_t
*ptr
, pgd_t val
)
752 u
.ptr
= virt_to_machine(ptr
).maddr
;
753 u
.val
= pgd_val_ma(val
);
754 xen_extend_mmu_update(&u
);
758 * Raw hypercall-based set_pgd, intended for in early boot before
759 * there's a page structure. This implies:
760 * 1. The only existing pagetable is the kernel's
761 * 2. It is always pinned
762 * 3. It has no user pagetable attached to it
764 void __init
xen_set_pgd_hyper(pgd_t
*ptr
, pgd_t val
)
770 __xen_set_pgd_hyper(ptr
, val
);
772 xen_mc_issue(PARAVIRT_LAZY_MMU
);
777 void xen_set_pgd(pgd_t
*ptr
, pgd_t val
)
779 pgd_t
*user_ptr
= xen_get_user_pgd(ptr
);
781 ADD_STATS(pgd_update
, 1);
783 /* If page is not pinned, we can just update the entry
785 if (!xen_page_pinned(ptr
)) {
788 WARN_ON(xen_page_pinned(user_ptr
));
794 ADD_STATS(pgd_update_pinned
, 1);
795 ADD_STATS(pgd_update_batched
, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU
);
797 /* If it's pinned, then we can at least batch the kernel and
798 user updates together. */
801 __xen_set_pgd_hyper(ptr
, val
);
803 __xen_set_pgd_hyper(user_ptr
, val
);
805 xen_mc_issue(PARAVIRT_LAZY_MMU
);
807 #endif /* PAGETABLE_LEVELS == 4 */
810 * (Yet another) pagetable walker. This one is intended for pinning a
811 * pagetable. This means that it walks a pagetable and calls the
812 * callback function on each page it finds making up the page table,
813 * at every level. It walks the entire pagetable, but it only bothers
814 * pinning pte pages which are below limit. In the normal case this
815 * will be STACK_TOP_MAX, but at boot we need to pin up to
818 * For 32-bit the important bit is that we don't pin beyond there,
819 * because then we start getting into Xen's ptes.
821 * For 64-bit, we must skip the Xen hole in the middle of the address
822 * space, just after the big x86-64 virtual hole.
824 static int __xen_pgd_walk(struct mm_struct
*mm
, pgd_t
*pgd
,
825 int (*func
)(struct mm_struct
*mm
, struct page
*,
830 unsigned hole_low
, hole_high
;
831 unsigned pgdidx_limit
, pudidx_limit
, pmdidx_limit
;
832 unsigned pgdidx
, pudidx
, pmdidx
;
834 /* The limit is the last byte to be touched */
836 BUG_ON(limit
>= FIXADDR_TOP
);
838 if (xen_feature(XENFEAT_auto_translated_physmap
))
842 * 64-bit has a great big hole in the middle of the address
843 * space, which contains the Xen mappings. On 32-bit these
844 * will end up making a zero-sized hole and so is a no-op.
846 hole_low
= pgd_index(USER_LIMIT
);
847 hole_high
= pgd_index(PAGE_OFFSET
);
849 pgdidx_limit
= pgd_index(limit
);
851 pudidx_limit
= pud_index(limit
);
856 pmdidx_limit
= pmd_index(limit
);
861 for (pgdidx
= 0; pgdidx
<= pgdidx_limit
; pgdidx
++) {
864 if (pgdidx
>= hole_low
&& pgdidx
< hole_high
)
867 if (!pgd_val(pgd
[pgdidx
]))
870 pud
= pud_offset(&pgd
[pgdidx
], 0);
872 if (PTRS_PER_PUD
> 1) /* not folded */
873 flush
|= (*func
)(mm
, virt_to_page(pud
), PT_PUD
);
875 for (pudidx
= 0; pudidx
< PTRS_PER_PUD
; pudidx
++) {
878 if (pgdidx
== pgdidx_limit
&&
879 pudidx
> pudidx_limit
)
882 if (pud_none(pud
[pudidx
]))
885 pmd
= pmd_offset(&pud
[pudidx
], 0);
887 if (PTRS_PER_PMD
> 1) /* not folded */
888 flush
|= (*func
)(mm
, virt_to_page(pmd
), PT_PMD
);
890 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
; pmdidx
++) {
893 if (pgdidx
== pgdidx_limit
&&
894 pudidx
== pudidx_limit
&&
895 pmdidx
> pmdidx_limit
)
898 if (pmd_none(pmd
[pmdidx
]))
901 pte
= pmd_page(pmd
[pmdidx
]);
902 flush
|= (*func
)(mm
, pte
, PT_PTE
);
908 /* Do the top level last, so that the callbacks can use it as
909 a cue to do final things like tlb flushes. */
910 flush
|= (*func
)(mm
, virt_to_page(pgd
), PT_PGD
);
915 static int xen_pgd_walk(struct mm_struct
*mm
,
916 int (*func
)(struct mm_struct
*mm
, struct page
*,
920 return __xen_pgd_walk(mm
, mm
->pgd
, func
, limit
);
923 /* If we're using split pte locks, then take the page's lock and
924 return a pointer to it. Otherwise return NULL. */
925 static spinlock_t
*xen_pte_lock(struct page
*page
, struct mm_struct
*mm
)
927 spinlock_t
*ptl
= NULL
;
929 #if USE_SPLIT_PTLOCKS
930 ptl
= __pte_lockptr(page
);
931 spin_lock_nest_lock(ptl
, &mm
->page_table_lock
);
937 static void xen_pte_unlock(void *v
)
943 static void xen_do_pin(unsigned level
, unsigned long pfn
)
945 struct mmuext_op
*op
;
946 struct multicall_space mcs
;
948 mcs
= __xen_mc_entry(sizeof(*op
));
951 op
->arg1
.mfn
= pfn_to_mfn(pfn
);
952 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
955 static int xen_pin_page(struct mm_struct
*mm
, struct page
*page
,
958 unsigned pgfl
= TestSetPagePinned(page
);
962 flush
= 0; /* already pinned */
963 else if (PageHighMem(page
))
964 /* kmaps need flushing if we found an unpinned
968 void *pt
= lowmem_page_address(page
);
969 unsigned long pfn
= page_to_pfn(page
);
970 struct multicall_space mcs
= __xen_mc_entry(0);
976 * We need to hold the pagetable lock between the time
977 * we make the pagetable RO and when we actually pin
978 * it. If we don't, then other users may come in and
979 * attempt to update the pagetable by writing it,
980 * which will fail because the memory is RO but not
981 * pinned, so Xen won't do the trap'n'emulate.
983 * If we're using split pte locks, we can't hold the
984 * entire pagetable's worth of locks during the
985 * traverse, because we may wrap the preempt count (8
986 * bits). The solution is to mark RO and pin each PTE
987 * page while holding the lock. This means the number
988 * of locks we end up holding is never more than a
989 * batch size (~32 entries, at present).
991 * If we're not using split pte locks, we needn't pin
992 * the PTE pages independently, because we're
993 * protected by the overall pagetable lock.
997 ptl
= xen_pte_lock(page
, mm
);
999 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
1000 pfn_pte(pfn
, PAGE_KERNEL_RO
),
1001 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
1004 xen_do_pin(MMUEXT_PIN_L1_TABLE
, pfn
);
1006 /* Queue a deferred unlock for when this batch
1008 xen_mc_callback(xen_pte_unlock
, ptl
);
1015 /* This is called just after a mm has been created, but it has not
1016 been used yet. We need to make sure that its pagetable is all
1017 read-only, and can be pinned. */
1018 static void __xen_pgd_pin(struct mm_struct
*mm
, pgd_t
*pgd
)
1022 if (__xen_pgd_walk(mm
, pgd
, xen_pin_page
, USER_LIMIT
)) {
1023 /* re-enable interrupts for flushing */
1026 kmap_flush_unused();
1031 #ifdef CONFIG_X86_64
1033 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
1035 xen_do_pin(MMUEXT_PIN_L4_TABLE
, PFN_DOWN(__pa(pgd
)));
1038 xen_pin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
1039 xen_do_pin(MMUEXT_PIN_L4_TABLE
,
1040 PFN_DOWN(__pa(user_pgd
)));
1043 #else /* CONFIG_X86_32 */
1044 #ifdef CONFIG_X86_PAE
1045 /* Need to make sure unshared kernel PMD is pinnable */
1046 xen_pin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
1049 xen_do_pin(MMUEXT_PIN_L3_TABLE
, PFN_DOWN(__pa(pgd
)));
1050 #endif /* CONFIG_X86_64 */
1054 static void xen_pgd_pin(struct mm_struct
*mm
)
1056 __xen_pgd_pin(mm
, mm
->pgd
);
1060 * On save, we need to pin all pagetables to make sure they get their
1061 * mfns turned into pfns. Search the list for any unpinned pgds and pin
1062 * them (unpinned pgds are not currently in use, probably because the
1063 * process is under construction or destruction).
1065 * Expected to be called in stop_machine() ("equivalent to taking
1066 * every spinlock in the system"), so the locking doesn't really
1067 * matter all that much.
1069 void xen_mm_pin_all(void)
1071 unsigned long flags
;
1074 spin_lock_irqsave(&pgd_lock
, flags
);
1076 list_for_each_entry(page
, &pgd_list
, lru
) {
1077 if (!PagePinned(page
)) {
1078 __xen_pgd_pin(&init_mm
, (pgd_t
*)page_address(page
));
1079 SetPageSavePinned(page
);
1083 spin_unlock_irqrestore(&pgd_lock
, flags
);
1087 * The init_mm pagetable is really pinned as soon as its created, but
1088 * that's before we have page structures to store the bits. So do all
1089 * the book-keeping now.
1091 static __init
int xen_mark_pinned(struct mm_struct
*mm
, struct page
*page
,
1092 enum pt_level level
)
1094 SetPagePinned(page
);
1098 static void __init
xen_mark_init_mm_pinned(void)
1100 xen_pgd_walk(&init_mm
, xen_mark_pinned
, FIXADDR_TOP
);
1103 static int xen_unpin_page(struct mm_struct
*mm
, struct page
*page
,
1104 enum pt_level level
)
1106 unsigned pgfl
= TestClearPagePinned(page
);
1108 if (pgfl
&& !PageHighMem(page
)) {
1109 void *pt
= lowmem_page_address(page
);
1110 unsigned long pfn
= page_to_pfn(page
);
1111 spinlock_t
*ptl
= NULL
;
1112 struct multicall_space mcs
;
1115 * Do the converse to pin_page. If we're using split
1116 * pte locks, we must be holding the lock for while
1117 * the pte page is unpinned but still RO to prevent
1118 * concurrent updates from seeing it in this
1119 * partially-pinned state.
1121 if (level
== PT_PTE
) {
1122 ptl
= xen_pte_lock(page
, mm
);
1125 xen_do_pin(MMUEXT_UNPIN_TABLE
, pfn
);
1128 mcs
= __xen_mc_entry(0);
1130 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
1131 pfn_pte(pfn
, PAGE_KERNEL
),
1132 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
1135 /* unlock when batch completed */
1136 xen_mc_callback(xen_pte_unlock
, ptl
);
1140 return 0; /* never need to flush on unpin */
1143 /* Release a pagetables pages back as normal RW */
1144 static void __xen_pgd_unpin(struct mm_struct
*mm
, pgd_t
*pgd
)
1148 xen_do_pin(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1150 #ifdef CONFIG_X86_64
1152 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
1155 xen_do_pin(MMUEXT_UNPIN_TABLE
,
1156 PFN_DOWN(__pa(user_pgd
)));
1157 xen_unpin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
1162 #ifdef CONFIG_X86_PAE
1163 /* Need to make sure unshared kernel PMD is unpinned */
1164 xen_unpin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
1168 __xen_pgd_walk(mm
, pgd
, xen_unpin_page
, USER_LIMIT
);
1173 static void xen_pgd_unpin(struct mm_struct
*mm
)
1175 __xen_pgd_unpin(mm
, mm
->pgd
);
1179 * On resume, undo any pinning done at save, so that the rest of the
1180 * kernel doesn't see any unexpected pinned pagetables.
1182 void xen_mm_unpin_all(void)
1184 unsigned long flags
;
1187 spin_lock_irqsave(&pgd_lock
, flags
);
1189 list_for_each_entry(page
, &pgd_list
, lru
) {
1190 if (PageSavePinned(page
)) {
1191 BUG_ON(!PagePinned(page
));
1192 __xen_pgd_unpin(&init_mm
, (pgd_t
*)page_address(page
));
1193 ClearPageSavePinned(page
);
1197 spin_unlock_irqrestore(&pgd_lock
, flags
);
1200 void xen_activate_mm(struct mm_struct
*prev
, struct mm_struct
*next
)
1202 spin_lock(&next
->page_table_lock
);
1204 spin_unlock(&next
->page_table_lock
);
1207 void xen_dup_mmap(struct mm_struct
*oldmm
, struct mm_struct
*mm
)
1209 spin_lock(&mm
->page_table_lock
);
1211 spin_unlock(&mm
->page_table_lock
);
1216 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1217 we need to repoint it somewhere else before we can unpin it. */
1218 static void drop_other_mm_ref(void *info
)
1220 struct mm_struct
*mm
= info
;
1221 struct mm_struct
*active_mm
;
1223 active_mm
= percpu_read(cpu_tlbstate
.active_mm
);
1225 if (active_mm
== mm
)
1226 leave_mm(smp_processor_id());
1228 /* If this cpu still has a stale cr3 reference, then make sure
1229 it has been flushed. */
1230 if (percpu_read(xen_current_cr3
) == __pa(mm
->pgd
))
1231 load_cr3(swapper_pg_dir
);
1234 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1239 if (current
->active_mm
== mm
) {
1240 if (current
->mm
== mm
)
1241 load_cr3(swapper_pg_dir
);
1243 leave_mm(smp_processor_id());
1246 /* Get the "official" set of cpus referring to our pagetable. */
1247 if (!alloc_cpumask_var(&mask
, GFP_ATOMIC
)) {
1248 for_each_online_cpu(cpu
) {
1249 if (!cpumask_test_cpu(cpu
, mm_cpumask(mm
))
1250 && per_cpu(xen_current_cr3
, cpu
) != __pa(mm
->pgd
))
1252 smp_call_function_single(cpu
, drop_other_mm_ref
, mm
, 1);
1256 cpumask_copy(mask
, mm_cpumask(mm
));
1258 /* It's possible that a vcpu may have a stale reference to our
1259 cr3, because its in lazy mode, and it hasn't yet flushed
1260 its set of pending hypercalls yet. In this case, we can
1261 look at its actual current cr3 value, and force it to flush
1263 for_each_online_cpu(cpu
) {
1264 if (per_cpu(xen_current_cr3
, cpu
) == __pa(mm
->pgd
))
1265 cpumask_set_cpu(cpu
, mask
);
1268 if (!cpumask_empty(mask
))
1269 smp_call_function_many(mask
, drop_other_mm_ref
, mm
, 1);
1270 free_cpumask_var(mask
);
1273 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1275 if (current
->active_mm
== mm
)
1276 load_cr3(swapper_pg_dir
);
1281 * While a process runs, Xen pins its pagetables, which means that the
1282 * hypervisor forces it to be read-only, and it controls all updates
1283 * to it. This means that all pagetable updates have to go via the
1284 * hypervisor, which is moderately expensive.
1286 * Since we're pulling the pagetable down, we switch to use init_mm,
1287 * unpin old process pagetable and mark it all read-write, which
1288 * allows further operations on it to be simple memory accesses.
1290 * The only subtle point is that another CPU may be still using the
1291 * pagetable because of lazy tlb flushing. This means we need need to
1292 * switch all CPUs off this pagetable before we can unpin it.
1294 void xen_exit_mmap(struct mm_struct
*mm
)
1296 get_cpu(); /* make sure we don't move around */
1297 xen_drop_mm_ref(mm
);
1300 spin_lock(&mm
->page_table_lock
);
1302 /* pgd may not be pinned in the error exit path of execve */
1303 if (xen_page_pinned(mm
->pgd
))
1306 spin_unlock(&mm
->page_table_lock
);
1309 static __init
void xen_pagetable_setup_start(pgd_t
*base
)
1313 static void xen_post_allocator_init(void);
1315 static __init
void xen_pagetable_setup_done(pgd_t
*base
)
1317 xen_setup_shared_info();
1318 xen_post_allocator_init();
1321 static void xen_write_cr2(unsigned long cr2
)
1323 percpu_read(xen_vcpu
)->arch
.cr2
= cr2
;
1326 static unsigned long xen_read_cr2(void)
1328 return percpu_read(xen_vcpu
)->arch
.cr2
;
1331 unsigned long xen_read_cr2_direct(void)
1333 return percpu_read(xen_vcpu_info
.arch
.cr2
);
1336 static void xen_flush_tlb(void)
1338 struct mmuext_op
*op
;
1339 struct multicall_space mcs
;
1343 mcs
= xen_mc_entry(sizeof(*op
));
1346 op
->cmd
= MMUEXT_TLB_FLUSH_LOCAL
;
1347 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1349 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1354 static void xen_flush_tlb_single(unsigned long addr
)
1356 struct mmuext_op
*op
;
1357 struct multicall_space mcs
;
1361 mcs
= xen_mc_entry(sizeof(*op
));
1363 op
->cmd
= MMUEXT_INVLPG_LOCAL
;
1364 op
->arg1
.linear_addr
= addr
& PAGE_MASK
;
1365 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1367 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1372 static void xen_flush_tlb_others(const struct cpumask
*cpus
,
1373 struct mm_struct
*mm
, unsigned long va
)
1376 struct mmuext_op op
;
1377 DECLARE_BITMAP(mask
, NR_CPUS
);
1379 struct multicall_space mcs
;
1381 if (cpumask_empty(cpus
))
1382 return; /* nothing to do */
1384 mcs
= xen_mc_entry(sizeof(*args
));
1386 args
->op
.arg2
.vcpumask
= to_cpumask(args
->mask
);
1388 /* Remove us, and any offline CPUS. */
1389 cpumask_and(to_cpumask(args
->mask
), cpus
, cpu_online_mask
);
1390 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args
->mask
));
1392 if (va
== TLB_FLUSH_ALL
) {
1393 args
->op
.cmd
= MMUEXT_TLB_FLUSH_MULTI
;
1395 args
->op
.cmd
= MMUEXT_INVLPG_MULTI
;
1396 args
->op
.arg1
.linear_addr
= va
;
1399 MULTI_mmuext_op(mcs
.mc
, &args
->op
, 1, NULL
, DOMID_SELF
);
1401 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1404 static unsigned long xen_read_cr3(void)
1406 return percpu_read(xen_cr3
);
1409 static void set_current_cr3(void *v
)
1411 percpu_write(xen_current_cr3
, (unsigned long)v
);
1414 static void __xen_write_cr3(bool kernel
, unsigned long cr3
)
1416 struct mmuext_op
*op
;
1417 struct multicall_space mcs
;
1421 mfn
= pfn_to_mfn(PFN_DOWN(cr3
));
1425 WARN_ON(mfn
== 0 && kernel
);
1427 mcs
= __xen_mc_entry(sizeof(*op
));
1430 op
->cmd
= kernel
? MMUEXT_NEW_BASEPTR
: MMUEXT_NEW_USER_BASEPTR
;
1433 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1436 percpu_write(xen_cr3
, cr3
);
1438 /* Update xen_current_cr3 once the batch has actually
1440 xen_mc_callback(set_current_cr3
, (void *)cr3
);
1444 static void xen_write_cr3(unsigned long cr3
)
1446 BUG_ON(preemptible());
1448 xen_mc_batch(); /* disables interrupts */
1450 /* Update while interrupts are disabled, so its atomic with
1452 percpu_write(xen_cr3
, cr3
);
1454 __xen_write_cr3(true, cr3
);
1456 #ifdef CONFIG_X86_64
1458 pgd_t
*user_pgd
= xen_get_user_pgd(__va(cr3
));
1460 __xen_write_cr3(false, __pa(user_pgd
));
1462 __xen_write_cr3(false, 0);
1466 xen_mc_issue(PARAVIRT_LAZY_CPU
); /* interrupts restored */
1469 static int xen_pgd_alloc(struct mm_struct
*mm
)
1471 pgd_t
*pgd
= mm
->pgd
;
1474 BUG_ON(PagePinned(virt_to_page(pgd
)));
1476 #ifdef CONFIG_X86_64
1478 struct page
*page
= virt_to_page(pgd
);
1481 BUG_ON(page
->private != 0);
1485 user_pgd
= (pgd_t
*)__get_free_page(GFP_KERNEL
| __GFP_ZERO
);
1486 page
->private = (unsigned long)user_pgd
;
1488 if (user_pgd
!= NULL
) {
1489 user_pgd
[pgd_index(VSYSCALL_START
)] =
1490 __pgd(__pa(level3_user_vsyscall
) | _PAGE_TABLE
);
1494 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd
))));
1501 static void xen_pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
)
1503 #ifdef CONFIG_X86_64
1504 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
1507 free_page((unsigned long)user_pgd
);
1511 #ifdef CONFIG_X86_32
1512 static __init pte_t
mask_rw_pte(pte_t
*ptep
, pte_t pte
)
1514 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1515 if (pte_val_ma(*ptep
) & _PAGE_PRESENT
)
1516 pte
= __pte_ma(((pte_val_ma(*ptep
) & _PAGE_RW
) | ~_PAGE_RW
) &
1522 /* Init-time set_pte while constructing initial pagetables, which
1523 doesn't allow RO pagetable pages to be remapped RW */
1524 static __init
void xen_set_pte_init(pte_t
*ptep
, pte_t pte
)
1526 pte
= mask_rw_pte(ptep
, pte
);
1528 xen_set_pte(ptep
, pte
);
1532 static void pin_pagetable_pfn(unsigned cmd
, unsigned long pfn
)
1534 struct mmuext_op op
;
1536 op
.arg1
.mfn
= pfn_to_mfn(pfn
);
1537 if (HYPERVISOR_mmuext_op(&op
, 1, NULL
, DOMID_SELF
))
1541 /* Early in boot, while setting up the initial pagetable, assume
1542 everything is pinned. */
1543 static __init
void xen_alloc_pte_init(struct mm_struct
*mm
, unsigned long pfn
)
1545 #ifdef CONFIG_FLATMEM
1546 BUG_ON(mem_map
); /* should only be used early */
1548 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1549 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1552 /* Used for pmd and pud */
1553 static __init
void xen_alloc_pmd_init(struct mm_struct
*mm
, unsigned long pfn
)
1555 #ifdef CONFIG_FLATMEM
1556 BUG_ON(mem_map
); /* should only be used early */
1558 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1561 /* Early release_pte assumes that all pts are pinned, since there's
1562 only init_mm and anything attached to that is pinned. */
1563 static __init
void xen_release_pte_init(unsigned long pfn
)
1565 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1566 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1569 static __init
void xen_release_pmd_init(unsigned long pfn
)
1571 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1574 /* This needs to make sure the new pte page is pinned iff its being
1575 attached to a pinned pagetable. */
1576 static void xen_alloc_ptpage(struct mm_struct
*mm
, unsigned long pfn
, unsigned level
)
1578 struct page
*page
= pfn_to_page(pfn
);
1580 if (PagePinned(virt_to_page(mm
->pgd
))) {
1581 SetPagePinned(page
);
1583 if (!PageHighMem(page
)) {
1584 make_lowmem_page_readonly(__va(PFN_PHYS((unsigned long)pfn
)));
1585 if (level
== PT_PTE
&& USE_SPLIT_PTLOCKS
)
1586 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1588 /* make sure there are no stray mappings of
1590 kmap_flush_unused();
1595 static void xen_alloc_pte(struct mm_struct
*mm
, unsigned long pfn
)
1597 xen_alloc_ptpage(mm
, pfn
, PT_PTE
);
1600 static void xen_alloc_pmd(struct mm_struct
*mm
, unsigned long pfn
)
1602 xen_alloc_ptpage(mm
, pfn
, PT_PMD
);
1605 /* This should never happen until we're OK to use struct page */
1606 static void xen_release_ptpage(unsigned long pfn
, unsigned level
)
1608 struct page
*page
= pfn_to_page(pfn
);
1610 if (PagePinned(page
)) {
1611 if (!PageHighMem(page
)) {
1612 if (level
== PT_PTE
&& USE_SPLIT_PTLOCKS
)
1613 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1614 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1616 ClearPagePinned(page
);
1620 static void xen_release_pte(unsigned long pfn
)
1622 xen_release_ptpage(pfn
, PT_PTE
);
1625 static void xen_release_pmd(unsigned long pfn
)
1627 xen_release_ptpage(pfn
, PT_PMD
);
1630 #if PAGETABLE_LEVELS == 4
1631 static void xen_alloc_pud(struct mm_struct
*mm
, unsigned long pfn
)
1633 xen_alloc_ptpage(mm
, pfn
, PT_PUD
);
1636 static void xen_release_pud(unsigned long pfn
)
1638 xen_release_ptpage(pfn
, PT_PUD
);
1642 void __init
xen_reserve_top(void)
1644 #ifdef CONFIG_X86_32
1645 unsigned long top
= HYPERVISOR_VIRT_START
;
1646 struct xen_platform_parameters pp
;
1648 if (HYPERVISOR_xen_version(XENVER_platform_parameters
, &pp
) == 0)
1649 top
= pp
.virt_start
;
1651 reserve_top_address(-top
);
1652 #endif /* CONFIG_X86_32 */
1656 * Like __va(), but returns address in the kernel mapping (which is
1657 * all we have until the physical memory mapping has been set up.
1659 static void *__ka(phys_addr_t paddr
)
1661 #ifdef CONFIG_X86_64
1662 return (void *)(paddr
+ __START_KERNEL_map
);
1668 /* Convert a machine address to physical address */
1669 static unsigned long m2p(phys_addr_t maddr
)
1673 maddr
&= PTE_PFN_MASK
;
1674 paddr
= mfn_to_pfn(maddr
>> PAGE_SHIFT
) << PAGE_SHIFT
;
1679 /* Convert a machine address to kernel virtual */
1680 static void *m2v(phys_addr_t maddr
)
1682 return __ka(m2p(maddr
));
1685 static void set_page_prot(void *addr
, pgprot_t prot
)
1687 unsigned long pfn
= __pa(addr
) >> PAGE_SHIFT
;
1688 pte_t pte
= pfn_pte(pfn
, prot
);
1690 if (HYPERVISOR_update_va_mapping((unsigned long)addr
, pte
, 0))
1694 static __init
void xen_map_identity_early(pmd_t
*pmd
, unsigned long max_pfn
)
1696 unsigned pmdidx
, pteidx
;
1702 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
&& pfn
< max_pfn
; pmdidx
++) {
1705 /* Reuse or allocate a page of ptes */
1706 if (pmd_present(pmd
[pmdidx
]))
1707 pte_page
= m2v(pmd
[pmdidx
].pmd
);
1709 /* Check for free pte pages */
1710 if (ident_pte
== ARRAY_SIZE(level1_ident_pgt
))
1713 pte_page
= &level1_ident_pgt
[ident_pte
];
1714 ident_pte
+= PTRS_PER_PTE
;
1716 pmd
[pmdidx
] = __pmd(__pa(pte_page
) | _PAGE_TABLE
);
1719 /* Install mappings */
1720 for (pteidx
= 0; pteidx
< PTRS_PER_PTE
; pteidx
++, pfn
++) {
1723 if (pfn
> max_pfn_mapped
)
1724 max_pfn_mapped
= pfn
;
1726 if (!pte_none(pte_page
[pteidx
]))
1729 pte
= pfn_pte(pfn
, PAGE_KERNEL_EXEC
);
1730 pte_page
[pteidx
] = pte
;
1734 for (pteidx
= 0; pteidx
< ident_pte
; pteidx
+= PTRS_PER_PTE
)
1735 set_page_prot(&level1_ident_pgt
[pteidx
], PAGE_KERNEL_RO
);
1737 set_page_prot(pmd
, PAGE_KERNEL_RO
);
1740 #ifdef CONFIG_X86_64
1741 static void convert_pfn_mfn(void *v
)
1746 /* All levels are converted the same way, so just treat them
1748 for (i
= 0; i
< PTRS_PER_PTE
; i
++)
1749 pte
[i
] = xen_make_pte(pte
[i
].pte
);
1753 * Set up the inital kernel pagetable.
1755 * We can construct this by grafting the Xen provided pagetable into
1756 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1757 * level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This
1758 * means that only the kernel has a physical mapping to start with -
1759 * but that's enough to get __va working. We need to fill in the rest
1760 * of the physical mapping once some sort of allocator has been set
1763 __init pgd_t
*xen_setup_kernel_pagetable(pgd_t
*pgd
,
1764 unsigned long max_pfn
)
1769 /* Zap identity mapping */
1770 init_level4_pgt
[0] = __pgd(0);
1772 /* Pre-constructed entries are in pfn, so convert to mfn */
1773 convert_pfn_mfn(init_level4_pgt
);
1774 convert_pfn_mfn(level3_ident_pgt
);
1775 convert_pfn_mfn(level3_kernel_pgt
);
1777 l3
= m2v(pgd
[pgd_index(__START_KERNEL_map
)].pgd
);
1778 l2
= m2v(l3
[pud_index(__START_KERNEL_map
)].pud
);
1780 memcpy(level2_ident_pgt
, l2
, sizeof(pmd_t
) * PTRS_PER_PMD
);
1781 memcpy(level2_kernel_pgt
, l2
, sizeof(pmd_t
) * PTRS_PER_PMD
);
1783 l3
= m2v(pgd
[pgd_index(__START_KERNEL_map
+ PMD_SIZE
)].pgd
);
1784 l2
= m2v(l3
[pud_index(__START_KERNEL_map
+ PMD_SIZE
)].pud
);
1785 memcpy(level2_fixmap_pgt
, l2
, sizeof(pmd_t
) * PTRS_PER_PMD
);
1787 /* Set up identity map */
1788 xen_map_identity_early(level2_ident_pgt
, max_pfn
);
1790 /* Make pagetable pieces RO */
1791 set_page_prot(init_level4_pgt
, PAGE_KERNEL_RO
);
1792 set_page_prot(level3_ident_pgt
, PAGE_KERNEL_RO
);
1793 set_page_prot(level3_kernel_pgt
, PAGE_KERNEL_RO
);
1794 set_page_prot(level3_user_vsyscall
, PAGE_KERNEL_RO
);
1795 set_page_prot(level2_kernel_pgt
, PAGE_KERNEL_RO
);
1796 set_page_prot(level2_fixmap_pgt
, PAGE_KERNEL_RO
);
1798 /* Pin down new L4 */
1799 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE
,
1800 PFN_DOWN(__pa_symbol(init_level4_pgt
)));
1802 /* Unpin Xen-provided one */
1803 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1806 pgd
= init_level4_pgt
;
1809 * At this stage there can be no user pgd, and no page
1810 * structure to attach it to, so make sure we just set kernel
1814 __xen_write_cr3(true, __pa(pgd
));
1815 xen_mc_issue(PARAVIRT_LAZY_CPU
);
1817 reserve_early(__pa(xen_start_info
->pt_base
),
1818 __pa(xen_start_info
->pt_base
+
1819 xen_start_info
->nr_pt_frames
* PAGE_SIZE
),
1824 #else /* !CONFIG_X86_64 */
1825 static pmd_t level2_kernel_pgt
[PTRS_PER_PMD
] __page_aligned_bss
;
1827 __init pgd_t
*xen_setup_kernel_pagetable(pgd_t
*pgd
,
1828 unsigned long max_pfn
)
1832 max_pfn_mapped
= PFN_DOWN(__pa(xen_start_info
->pt_base
) +
1833 xen_start_info
->nr_pt_frames
* PAGE_SIZE
+
1836 kernel_pmd
= m2v(pgd
[KERNEL_PGD_BOUNDARY
].pgd
);
1837 memcpy(level2_kernel_pgt
, kernel_pmd
, sizeof(pmd_t
) * PTRS_PER_PMD
);
1839 xen_map_identity_early(level2_kernel_pgt
, max_pfn
);
1841 memcpy(swapper_pg_dir
, pgd
, sizeof(pgd_t
) * PTRS_PER_PGD
);
1842 set_pgd(&swapper_pg_dir
[KERNEL_PGD_BOUNDARY
],
1843 __pgd(__pa(level2_kernel_pgt
) | _PAGE_PRESENT
));
1845 set_page_prot(level2_kernel_pgt
, PAGE_KERNEL_RO
);
1846 set_page_prot(swapper_pg_dir
, PAGE_KERNEL_RO
);
1847 set_page_prot(empty_zero_page
, PAGE_KERNEL_RO
);
1849 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1851 xen_write_cr3(__pa(swapper_pg_dir
));
1853 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
, PFN_DOWN(__pa(swapper_pg_dir
)));
1855 reserve_early(__pa(xen_start_info
->pt_base
),
1856 __pa(xen_start_info
->pt_base
+
1857 xen_start_info
->nr_pt_frames
* PAGE_SIZE
),
1860 return swapper_pg_dir
;
1862 #endif /* CONFIG_X86_64 */
1864 static void xen_set_fixmap(unsigned idx
, phys_addr_t phys
, pgprot_t prot
)
1868 phys
>>= PAGE_SHIFT
;
1871 case FIX_BTMAP_END
... FIX_BTMAP_BEGIN
:
1872 #ifdef CONFIG_X86_F00F_BUG
1875 #ifdef CONFIG_X86_32
1878 # ifdef CONFIG_HIGHMEM
1879 case FIX_KMAP_BEGIN
... FIX_KMAP_END
:
1882 case VSYSCALL_LAST_PAGE
... VSYSCALL_FIRST_PAGE
:
1884 #ifdef CONFIG_X86_LOCAL_APIC
1885 case FIX_APIC_BASE
: /* maps dummy local APIC */
1887 case FIX_TEXT_POKE0
:
1888 case FIX_TEXT_POKE1
:
1889 /* All local page mappings */
1890 pte
= pfn_pte(phys
, prot
);
1893 case FIX_PARAVIRT_BOOTMAP
:
1894 /* This is an MFN, but it isn't an IO mapping from the
1896 pte
= mfn_pte(phys
, prot
);
1900 /* By default, set_fixmap is used for hardware mappings */
1901 pte
= mfn_pte(phys
, __pgprot(pgprot_val(prot
) | _PAGE_IOMAP
));
1905 __native_set_fixmap(idx
, pte
);
1907 #ifdef CONFIG_X86_64
1908 /* Replicate changes to map the vsyscall page into the user
1909 pagetable vsyscall mapping. */
1910 if (idx
>= VSYSCALL_LAST_PAGE
&& idx
<= VSYSCALL_FIRST_PAGE
) {
1911 unsigned long vaddr
= __fix_to_virt(idx
);
1912 set_pte_vaddr_pud(level3_user_vsyscall
, vaddr
, pte
);
1917 static __init
void xen_post_allocator_init(void)
1919 pv_mmu_ops
.set_pte
= xen_set_pte
;
1920 pv_mmu_ops
.set_pmd
= xen_set_pmd
;
1921 pv_mmu_ops
.set_pud
= xen_set_pud
;
1922 #if PAGETABLE_LEVELS == 4
1923 pv_mmu_ops
.set_pgd
= xen_set_pgd
;
1926 /* This will work as long as patching hasn't happened yet
1927 (which it hasn't) */
1928 pv_mmu_ops
.alloc_pte
= xen_alloc_pte
;
1929 pv_mmu_ops
.alloc_pmd
= xen_alloc_pmd
;
1930 pv_mmu_ops
.release_pte
= xen_release_pte
;
1931 pv_mmu_ops
.release_pmd
= xen_release_pmd
;
1932 #if PAGETABLE_LEVELS == 4
1933 pv_mmu_ops
.alloc_pud
= xen_alloc_pud
;
1934 pv_mmu_ops
.release_pud
= xen_release_pud
;
1937 #ifdef CONFIG_X86_64
1938 SetPagePinned(virt_to_page(level3_user_vsyscall
));
1940 xen_mark_init_mm_pinned();
1943 static void xen_leave_lazy_mmu(void)
1947 paravirt_leave_lazy_mmu();
1951 static const struct pv_mmu_ops xen_mmu_ops __initdata
= {
1952 .read_cr2
= xen_read_cr2
,
1953 .write_cr2
= xen_write_cr2
,
1955 .read_cr3
= xen_read_cr3
,
1956 .write_cr3
= xen_write_cr3
,
1958 .flush_tlb_user
= xen_flush_tlb
,
1959 .flush_tlb_kernel
= xen_flush_tlb
,
1960 .flush_tlb_single
= xen_flush_tlb_single
,
1961 .flush_tlb_others
= xen_flush_tlb_others
,
1963 .pte_update
= paravirt_nop
,
1964 .pte_update_defer
= paravirt_nop
,
1966 .pgd_alloc
= xen_pgd_alloc
,
1967 .pgd_free
= xen_pgd_free
,
1969 .alloc_pte
= xen_alloc_pte_init
,
1970 .release_pte
= xen_release_pte_init
,
1971 .alloc_pmd
= xen_alloc_pmd_init
,
1972 .alloc_pmd_clone
= paravirt_nop
,
1973 .release_pmd
= xen_release_pmd_init
,
1975 #ifdef CONFIG_X86_64
1976 .set_pte
= xen_set_pte
,
1978 .set_pte
= xen_set_pte_init
,
1980 .set_pte_at
= xen_set_pte_at
,
1981 .set_pmd
= xen_set_pmd_hyper
,
1983 .ptep_modify_prot_start
= __ptep_modify_prot_start
,
1984 .ptep_modify_prot_commit
= __ptep_modify_prot_commit
,
1986 .pte_val
= PV_CALLEE_SAVE(xen_pte_val
),
1987 .pgd_val
= PV_CALLEE_SAVE(xen_pgd_val
),
1989 .make_pte
= PV_CALLEE_SAVE(xen_make_pte
),
1990 .make_pgd
= PV_CALLEE_SAVE(xen_make_pgd
),
1992 #ifdef CONFIG_X86_PAE
1993 .set_pte_atomic
= xen_set_pte_atomic
,
1994 .pte_clear
= xen_pte_clear
,
1995 .pmd_clear
= xen_pmd_clear
,
1996 #endif /* CONFIG_X86_PAE */
1997 .set_pud
= xen_set_pud_hyper
,
1999 .make_pmd
= PV_CALLEE_SAVE(xen_make_pmd
),
2000 .pmd_val
= PV_CALLEE_SAVE(xen_pmd_val
),
2002 #if PAGETABLE_LEVELS == 4
2003 .pud_val
= PV_CALLEE_SAVE(xen_pud_val
),
2004 .make_pud
= PV_CALLEE_SAVE(xen_make_pud
),
2005 .set_pgd
= xen_set_pgd_hyper
,
2007 .alloc_pud
= xen_alloc_pmd_init
,
2008 .release_pud
= xen_release_pmd_init
,
2009 #endif /* PAGETABLE_LEVELS == 4 */
2011 .activate_mm
= xen_activate_mm
,
2012 .dup_mmap
= xen_dup_mmap
,
2013 .exit_mmap
= xen_exit_mmap
,
2016 .enter
= paravirt_enter_lazy_mmu
,
2017 .leave
= xen_leave_lazy_mmu
,
2020 .set_fixmap
= xen_set_fixmap
,
2023 void __init
xen_init_mmu_ops(void)
2025 x86_init
.paging
.pagetable_setup_start
= xen_pagetable_setup_start
;
2026 x86_init
.paging
.pagetable_setup_done
= xen_pagetable_setup_done
;
2027 pv_mmu_ops
= xen_mmu_ops
;
2029 vmap_lazy_unmap
= false;
2032 /* Protected by xen_reservation_lock. */
2033 #define MAX_CONTIG_ORDER 9 /* 2MB */
2034 static unsigned long discontig_frames
[1<<MAX_CONTIG_ORDER
];
2036 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2037 static void xen_zap_pfn_range(unsigned long vaddr
, unsigned int order
,
2038 unsigned long *in_frames
,
2039 unsigned long *out_frames
)
2042 struct multicall_space mcs
;
2045 for (i
= 0; i
< (1UL<<order
); i
++, vaddr
+= PAGE_SIZE
) {
2046 mcs
= __xen_mc_entry(0);
2049 in_frames
[i
] = virt_to_mfn(vaddr
);
2051 MULTI_update_va_mapping(mcs
.mc
, vaddr
, VOID_PTE
, 0);
2052 set_phys_to_machine(virt_to_pfn(vaddr
), INVALID_P2M_ENTRY
);
2055 out_frames
[i
] = virt_to_pfn(vaddr
);
2061 * Update the pfn-to-mfn mappings for a virtual address range, either to
2062 * point to an array of mfns, or contiguously from a single starting
2065 static void xen_remap_exchanged_ptes(unsigned long vaddr
, int order
,
2066 unsigned long *mfns
,
2067 unsigned long first_mfn
)
2074 limit
= 1u << order
;
2075 for (i
= 0; i
< limit
; i
++, vaddr
+= PAGE_SIZE
) {
2076 struct multicall_space mcs
;
2079 mcs
= __xen_mc_entry(0);
2083 mfn
= first_mfn
+ i
;
2085 if (i
< (limit
- 1))
2089 flags
= UVMF_INVLPG
| UVMF_ALL
;
2091 flags
= UVMF_TLB_FLUSH
| UVMF_ALL
;
2094 MULTI_update_va_mapping(mcs
.mc
, vaddr
,
2095 mfn_pte(mfn
, PAGE_KERNEL
), flags
);
2097 set_phys_to_machine(virt_to_pfn(vaddr
), mfn
);
2104 * Perform the hypercall to exchange a region of our pfns to point to
2105 * memory with the required contiguous alignment. Takes the pfns as
2106 * input, and populates mfns as output.
2108 * Returns a success code indicating whether the hypervisor was able to
2109 * satisfy the request or not.
2111 static int xen_exchange_memory(unsigned long extents_in
, unsigned int order_in
,
2112 unsigned long *pfns_in
,
2113 unsigned long extents_out
,
2114 unsigned int order_out
,
2115 unsigned long *mfns_out
,
2116 unsigned int address_bits
)
2121 struct xen_memory_exchange exchange
= {
2123 .nr_extents
= extents_in
,
2124 .extent_order
= order_in
,
2125 .extent_start
= pfns_in
,
2129 .nr_extents
= extents_out
,
2130 .extent_order
= order_out
,
2131 .extent_start
= mfns_out
,
2132 .address_bits
= address_bits
,
2137 BUG_ON(extents_in
<< order_in
!= extents_out
<< order_out
);
2139 rc
= HYPERVISOR_memory_op(XENMEM_exchange
, &exchange
);
2140 success
= (exchange
.nr_exchanged
== extents_in
);
2142 BUG_ON(!success
&& ((exchange
.nr_exchanged
!= 0) || (rc
== 0)));
2143 BUG_ON(success
&& (rc
!= 0));
2148 int xen_create_contiguous_region(unsigned long vstart
, unsigned int order
,
2149 unsigned int address_bits
)
2151 unsigned long *in_frames
= discontig_frames
, out_frame
;
2152 unsigned long flags
;
2156 * Currently an auto-translated guest will not perform I/O, nor will
2157 * it require PAE page directories below 4GB. Therefore any calls to
2158 * this function are redundant and can be ignored.
2161 if (xen_feature(XENFEAT_auto_translated_physmap
))
2164 if (unlikely(order
> MAX_CONTIG_ORDER
))
2167 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2169 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2171 /* 1. Zap current PTEs, remembering MFNs. */
2172 xen_zap_pfn_range(vstart
, order
, in_frames
, NULL
);
2174 /* 2. Get a new contiguous memory extent. */
2175 out_frame
= virt_to_pfn(vstart
);
2176 success
= xen_exchange_memory(1UL << order
, 0, in_frames
,
2177 1, order
, &out_frame
,
2180 /* 3. Map the new extent in place of old pages. */
2182 xen_remap_exchanged_ptes(vstart
, order
, NULL
, out_frame
);
2184 xen_remap_exchanged_ptes(vstart
, order
, in_frames
, 0);
2186 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2188 return success
? 0 : -ENOMEM
;
2190 EXPORT_SYMBOL_GPL(xen_create_contiguous_region
);
2192 void xen_destroy_contiguous_region(unsigned long vstart
, unsigned int order
)
2194 unsigned long *out_frames
= discontig_frames
, in_frame
;
2195 unsigned long flags
;
2198 if (xen_feature(XENFEAT_auto_translated_physmap
))
2201 if (unlikely(order
> MAX_CONTIG_ORDER
))
2204 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2206 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2208 /* 1. Find start MFN of contiguous extent. */
2209 in_frame
= virt_to_mfn(vstart
);
2211 /* 2. Zap current PTEs. */
2212 xen_zap_pfn_range(vstart
, order
, NULL
, out_frames
);
2214 /* 3. Do the exchange for non-contiguous MFNs. */
2215 success
= xen_exchange_memory(1, order
, &in_frame
, 1UL << order
,
2218 /* 4. Map new pages in place of old pages. */
2220 xen_remap_exchanged_ptes(vstart
, order
, out_frames
, 0);
2222 xen_remap_exchanged_ptes(vstart
, order
, NULL
, in_frame
);
2224 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2226 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region
);
2228 #ifdef CONFIG_XEN_PVHVM
2229 static void xen_hvm_exit_mmap(struct mm_struct
*mm
)
2231 struct xen_hvm_pagetable_dying a
;
2234 a
.domid
= DOMID_SELF
;
2235 a
.gpa
= __pa(mm
->pgd
);
2236 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2237 WARN_ON_ONCE(rc
< 0);
2240 static int is_pagetable_dying_supported(void)
2242 struct xen_hvm_pagetable_dying a
;
2245 a
.domid
= DOMID_SELF
;
2247 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2249 printk(KERN_DEBUG
"HVMOP_pagetable_dying not supported\n");
2255 void __init
xen_hvm_init_mmu_ops(void)
2257 if (is_pagetable_dying_supported())
2258 pv_mmu_ops
.exit_mmap
= xen_hvm_exit_mmap
;
2262 #ifdef CONFIG_XEN_DEBUG_FS
2264 static struct dentry
*d_mmu_debug
;
2266 static int __init
xen_mmu_debugfs(void)
2268 struct dentry
*d_xen
= xen_init_debugfs();
2273 d_mmu_debug
= debugfs_create_dir("mmu", d_xen
);
2275 debugfs_create_u8("zero_stats", 0644, d_mmu_debug
, &zero_stats
);
2277 debugfs_create_u32("pgd_update", 0444, d_mmu_debug
, &mmu_stats
.pgd_update
);
2278 debugfs_create_u32("pgd_update_pinned", 0444, d_mmu_debug
,
2279 &mmu_stats
.pgd_update_pinned
);
2280 debugfs_create_u32("pgd_update_batched", 0444, d_mmu_debug
,
2281 &mmu_stats
.pgd_update_pinned
);
2283 debugfs_create_u32("pud_update", 0444, d_mmu_debug
, &mmu_stats
.pud_update
);
2284 debugfs_create_u32("pud_update_pinned", 0444, d_mmu_debug
,
2285 &mmu_stats
.pud_update_pinned
);
2286 debugfs_create_u32("pud_update_batched", 0444, d_mmu_debug
,
2287 &mmu_stats
.pud_update_pinned
);
2289 debugfs_create_u32("pmd_update", 0444, d_mmu_debug
, &mmu_stats
.pmd_update
);
2290 debugfs_create_u32("pmd_update_pinned", 0444, d_mmu_debug
,
2291 &mmu_stats
.pmd_update_pinned
);
2292 debugfs_create_u32("pmd_update_batched", 0444, d_mmu_debug
,
2293 &mmu_stats
.pmd_update_pinned
);
2295 debugfs_create_u32("pte_update", 0444, d_mmu_debug
, &mmu_stats
.pte_update
);
2296 // debugfs_create_u32("pte_update_pinned", 0444, d_mmu_debug,
2297 // &mmu_stats.pte_update_pinned);
2298 debugfs_create_u32("pte_update_batched", 0444, d_mmu_debug
,
2299 &mmu_stats
.pte_update_pinned
);
2301 debugfs_create_u32("mmu_update", 0444, d_mmu_debug
, &mmu_stats
.mmu_update
);
2302 debugfs_create_u32("mmu_update_extended", 0444, d_mmu_debug
,
2303 &mmu_stats
.mmu_update_extended
);
2304 xen_debugfs_create_u32_array("mmu_update_histo", 0444, d_mmu_debug
,
2305 mmu_stats
.mmu_update_histo
, 20);
2307 debugfs_create_u32("set_pte_at", 0444, d_mmu_debug
, &mmu_stats
.set_pte_at
);
2308 debugfs_create_u32("set_pte_at_batched", 0444, d_mmu_debug
,
2309 &mmu_stats
.set_pte_at_batched
);
2310 debugfs_create_u32("set_pte_at_current", 0444, d_mmu_debug
,
2311 &mmu_stats
.set_pte_at_current
);
2312 debugfs_create_u32("set_pte_at_kernel", 0444, d_mmu_debug
,
2313 &mmu_stats
.set_pte_at_kernel
);
2315 debugfs_create_u32("prot_commit", 0444, d_mmu_debug
, &mmu_stats
.prot_commit
);
2316 debugfs_create_u32("prot_commit_batched", 0444, d_mmu_debug
,
2317 &mmu_stats
.prot_commit_batched
);
2321 fs_initcall(xen_mmu_debugfs
);
2323 #endif /* CONFIG_XEN_DEBUG_FS */