2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
7 * Copyright (C) 2006 Qumranet, Inc.
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
20 #include <linux/kvm.h>
21 #include <linux/module.h>
22 #include <linux/errno.h>
23 #include <asm/processor.h>
24 #include <linux/percpu.h>
25 #include <linux/gfp.h>
28 #include <linux/miscdevice.h>
29 #include <linux/vmalloc.h>
30 #include <asm/uaccess.h>
31 #include <linux/reboot.h>
33 #include <linux/debugfs.h>
34 #include <linux/highmem.h>
35 #include <linux/file.h>
37 #include <linux/sysdev.h>
38 #include <linux/cpu.h>
40 #include <linux/mount.h>
42 #include "x86_emulate.h"
43 #include "segment_descriptor.h"
45 MODULE_AUTHOR("Qumranet");
46 MODULE_LICENSE("GPL");
48 static DEFINE_SPINLOCK(kvm_lock
);
49 static LIST_HEAD(vm_list
);
51 struct kvm_arch_ops
*kvm_arch_ops
;
52 struct kvm_stat kvm_stat
;
53 EXPORT_SYMBOL_GPL(kvm_stat
);
55 static struct kvm_stats_debugfs_item
{
58 struct dentry
*dentry
;
59 } debugfs_entries
[] = {
60 { "pf_fixed", &kvm_stat
.pf_fixed
},
61 { "pf_guest", &kvm_stat
.pf_guest
},
62 { "tlb_flush", &kvm_stat
.tlb_flush
},
63 { "invlpg", &kvm_stat
.invlpg
},
64 { "exits", &kvm_stat
.exits
},
65 { "io_exits", &kvm_stat
.io_exits
},
66 { "mmio_exits", &kvm_stat
.mmio_exits
},
67 { "signal_exits", &kvm_stat
.signal_exits
},
68 { "irq_window", &kvm_stat
.irq_window_exits
},
69 { "halt_exits", &kvm_stat
.halt_exits
},
70 { "request_irq", &kvm_stat
.request_irq_exits
},
71 { "irq_exits", &kvm_stat
.irq_exits
},
75 static struct dentry
*debugfs_dir
;
77 #define KVMFS_MAGIC 0x19700426
78 struct vfsmount
*kvmfs_mnt
;
80 #define MAX_IO_MSRS 256
82 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
83 #define LMSW_GUEST_MASK 0x0eULL
84 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
85 #define CR8_RESEVED_BITS (~0x0fULL)
86 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
89 // LDT or TSS descriptor in the GDT. 16 bytes.
90 struct segment_descriptor_64
{
91 struct segment_descriptor s
;
98 unsigned long segment_base(u16 selector
)
100 struct descriptor_table gdt
;
101 struct segment_descriptor
*d
;
102 unsigned long table_base
;
103 typedef unsigned long ul
;
109 asm ("sgdt %0" : "=m"(gdt
));
110 table_base
= gdt
.base
;
112 if (selector
& 4) { /* from ldt */
115 asm ("sldt %0" : "=g"(ldt_selector
));
116 table_base
= segment_base(ldt_selector
);
118 d
= (struct segment_descriptor
*)(table_base
+ (selector
& ~7));
119 v
= d
->base_low
| ((ul
)d
->base_mid
<< 16) | ((ul
)d
->base_high
<< 24);
122 && (d
->type
== 2 || d
->type
== 9 || d
->type
== 11))
123 v
|= ((ul
)((struct segment_descriptor_64
*)d
)->base_higher
) << 32;
127 EXPORT_SYMBOL_GPL(segment_base
);
129 static inline int valid_vcpu(int n
)
131 return likely(n
>= 0 && n
< KVM_MAX_VCPUS
);
134 int kvm_read_guest(struct kvm_vcpu
*vcpu
, gva_t addr
, unsigned long size
,
137 unsigned char *host_buf
= dest
;
138 unsigned long req_size
= size
;
146 paddr
= gva_to_hpa(vcpu
, addr
);
148 if (is_error_hpa(paddr
))
151 guest_buf
= (hva_t
)kmap_atomic(
152 pfn_to_page(paddr
>> PAGE_SHIFT
),
154 offset
= addr
& ~PAGE_MASK
;
156 now
= min(size
, PAGE_SIZE
- offset
);
157 memcpy(host_buf
, (void*)guest_buf
, now
);
161 kunmap_atomic((void *)(guest_buf
& PAGE_MASK
), KM_USER0
);
163 return req_size
- size
;
165 EXPORT_SYMBOL_GPL(kvm_read_guest
);
167 int kvm_write_guest(struct kvm_vcpu
*vcpu
, gva_t addr
, unsigned long size
,
170 unsigned char *host_buf
= data
;
171 unsigned long req_size
= size
;
179 paddr
= gva_to_hpa(vcpu
, addr
);
181 if (is_error_hpa(paddr
))
184 guest_buf
= (hva_t
)kmap_atomic(
185 pfn_to_page(paddr
>> PAGE_SHIFT
), KM_USER0
);
186 offset
= addr
& ~PAGE_MASK
;
188 now
= min(size
, PAGE_SIZE
- offset
);
189 memcpy((void*)guest_buf
, host_buf
, now
);
193 kunmap_atomic((void *)(guest_buf
& PAGE_MASK
), KM_USER0
);
195 return req_size
- size
;
197 EXPORT_SYMBOL_GPL(kvm_write_guest
);
199 static int vcpu_slot(struct kvm_vcpu
*vcpu
)
201 return vcpu
- vcpu
->kvm
->vcpus
;
205 * Switches to specified vcpu, until a matching vcpu_put()
207 static struct kvm_vcpu
*vcpu_load(struct kvm
*kvm
, int vcpu_slot
)
209 struct kvm_vcpu
*vcpu
= &kvm
->vcpus
[vcpu_slot
];
211 mutex_lock(&vcpu
->mutex
);
212 if (unlikely(!vcpu
->vmcs
)) {
213 mutex_unlock(&vcpu
->mutex
);
216 return kvm_arch_ops
->vcpu_load(vcpu
);
219 static void vcpu_put(struct kvm_vcpu
*vcpu
)
221 kvm_arch_ops
->vcpu_put(vcpu
);
222 mutex_unlock(&vcpu
->mutex
);
225 static int kvm_dev_open(struct inode
*inode
, struct file
*filp
)
227 struct kvm
*kvm
= kzalloc(sizeof(struct kvm
), GFP_KERNEL
);
233 spin_lock_init(&kvm
->lock
);
234 INIT_LIST_HEAD(&kvm
->active_mmu_pages
);
235 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
236 struct kvm_vcpu
*vcpu
= &kvm
->vcpus
[i
];
238 mutex_init(&vcpu
->mutex
);
241 vcpu
->mmu
.root_hpa
= INVALID_PAGE
;
242 INIT_LIST_HEAD(&vcpu
->free_pages
);
243 spin_lock(&kvm_lock
);
244 list_add(&kvm
->vm_list
, &vm_list
);
245 spin_unlock(&kvm_lock
);
247 filp
->private_data
= kvm
;
252 * Free any memory in @free but not in @dont.
254 static void kvm_free_physmem_slot(struct kvm_memory_slot
*free
,
255 struct kvm_memory_slot
*dont
)
259 if (!dont
|| free
->phys_mem
!= dont
->phys_mem
)
260 if (free
->phys_mem
) {
261 for (i
= 0; i
< free
->npages
; ++i
)
262 if (free
->phys_mem
[i
])
263 __free_page(free
->phys_mem
[i
]);
264 vfree(free
->phys_mem
);
267 if (!dont
|| free
->dirty_bitmap
!= dont
->dirty_bitmap
)
268 vfree(free
->dirty_bitmap
);
270 free
->phys_mem
= NULL
;
272 free
->dirty_bitmap
= NULL
;
275 static void kvm_free_physmem(struct kvm
*kvm
)
279 for (i
= 0; i
< kvm
->nmemslots
; ++i
)
280 kvm_free_physmem_slot(&kvm
->memslots
[i
], NULL
);
283 static void kvm_free_vcpu(struct kvm_vcpu
*vcpu
)
285 if (!vcpu_load(vcpu
->kvm
, vcpu_slot(vcpu
)))
288 kvm_mmu_destroy(vcpu
);
290 kvm_arch_ops
->vcpu_free(vcpu
);
293 static void kvm_free_vcpus(struct kvm
*kvm
)
297 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
)
298 kvm_free_vcpu(&kvm
->vcpus
[i
]);
301 static int kvm_dev_release(struct inode
*inode
, struct file
*filp
)
303 struct kvm
*kvm
= filp
->private_data
;
305 spin_lock(&kvm_lock
);
306 list_del(&kvm
->vm_list
);
307 spin_unlock(&kvm_lock
);
309 kvm_free_physmem(kvm
);
314 static void inject_gp(struct kvm_vcpu
*vcpu
)
316 kvm_arch_ops
->inject_gp(vcpu
, 0);
320 * Load the pae pdptrs. Return true is they are all valid.
322 static int load_pdptrs(struct kvm_vcpu
*vcpu
, unsigned long cr3
)
324 gfn_t pdpt_gfn
= cr3
>> PAGE_SHIFT
;
325 unsigned offset
= ((cr3
& (PAGE_SIZE
-1)) >> 5) << 2;
330 struct kvm_memory_slot
*memslot
;
332 spin_lock(&vcpu
->kvm
->lock
);
333 memslot
= gfn_to_memslot(vcpu
->kvm
, pdpt_gfn
);
334 /* FIXME: !memslot - emulate? 0xff? */
335 pdpt
= kmap_atomic(gfn_to_page(memslot
, pdpt_gfn
), KM_USER0
);
338 for (i
= 0; i
< 4; ++i
) {
339 pdpte
= pdpt
[offset
+ i
];
340 if ((pdpte
& 1) && (pdpte
& 0xfffffff0000001e6ull
)) {
346 for (i
= 0; i
< 4; ++i
)
347 vcpu
->pdptrs
[i
] = pdpt
[offset
+ i
];
350 kunmap_atomic(pdpt
, KM_USER0
);
351 spin_unlock(&vcpu
->kvm
->lock
);
356 void set_cr0(struct kvm_vcpu
*vcpu
, unsigned long cr0
)
358 if (cr0
& CR0_RESEVED_BITS
) {
359 printk(KERN_DEBUG
"set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
365 if ((cr0
& CR0_NW_MASK
) && !(cr0
& CR0_CD_MASK
)) {
366 printk(KERN_DEBUG
"set_cr0: #GP, CD == 0 && NW == 1\n");
371 if ((cr0
& CR0_PG_MASK
) && !(cr0
& CR0_PE_MASK
)) {
372 printk(KERN_DEBUG
"set_cr0: #GP, set PG flag "
373 "and a clear PE flag\n");
378 if (!is_paging(vcpu
) && (cr0
& CR0_PG_MASK
)) {
380 if ((vcpu
->shadow_efer
& EFER_LME
)) {
384 printk(KERN_DEBUG
"set_cr0: #GP, start paging "
385 "in long mode while PAE is disabled\n");
389 kvm_arch_ops
->get_cs_db_l_bits(vcpu
, &cs_db
, &cs_l
);
391 printk(KERN_DEBUG
"set_cr0: #GP, start paging "
392 "in long mode while CS.L == 1\n");
399 if (is_pae(vcpu
) && !load_pdptrs(vcpu
, vcpu
->cr3
)) {
400 printk(KERN_DEBUG
"set_cr0: #GP, pdptrs "
408 kvm_arch_ops
->set_cr0(vcpu
, cr0
);
411 spin_lock(&vcpu
->kvm
->lock
);
412 kvm_mmu_reset_context(vcpu
);
413 spin_unlock(&vcpu
->kvm
->lock
);
416 EXPORT_SYMBOL_GPL(set_cr0
);
418 void lmsw(struct kvm_vcpu
*vcpu
, unsigned long msw
)
420 kvm_arch_ops
->decache_cr0_cr4_guest_bits(vcpu
);
421 set_cr0(vcpu
, (vcpu
->cr0
& ~0x0ful
) | (msw
& 0x0f));
423 EXPORT_SYMBOL_GPL(lmsw
);
425 void set_cr4(struct kvm_vcpu
*vcpu
, unsigned long cr4
)
427 if (cr4
& CR4_RESEVED_BITS
) {
428 printk(KERN_DEBUG
"set_cr4: #GP, reserved bits\n");
433 if (is_long_mode(vcpu
)) {
434 if (!(cr4
& CR4_PAE_MASK
)) {
435 printk(KERN_DEBUG
"set_cr4: #GP, clearing PAE while "
440 } else if (is_paging(vcpu
) && !is_pae(vcpu
) && (cr4
& CR4_PAE_MASK
)
441 && !load_pdptrs(vcpu
, vcpu
->cr3
)) {
442 printk(KERN_DEBUG
"set_cr4: #GP, pdptrs reserved bits\n");
446 if (cr4
& CR4_VMXE_MASK
) {
447 printk(KERN_DEBUG
"set_cr4: #GP, setting VMXE\n");
451 kvm_arch_ops
->set_cr4(vcpu
, cr4
);
452 spin_lock(&vcpu
->kvm
->lock
);
453 kvm_mmu_reset_context(vcpu
);
454 spin_unlock(&vcpu
->kvm
->lock
);
456 EXPORT_SYMBOL_GPL(set_cr4
);
458 void set_cr3(struct kvm_vcpu
*vcpu
, unsigned long cr3
)
460 if (is_long_mode(vcpu
)) {
461 if (cr3
& CR3_L_MODE_RESEVED_BITS
) {
462 printk(KERN_DEBUG
"set_cr3: #GP, reserved bits\n");
467 if (cr3
& CR3_RESEVED_BITS
) {
468 printk(KERN_DEBUG
"set_cr3: #GP, reserved bits\n");
472 if (is_paging(vcpu
) && is_pae(vcpu
) &&
473 !load_pdptrs(vcpu
, cr3
)) {
474 printk(KERN_DEBUG
"set_cr3: #GP, pdptrs "
482 spin_lock(&vcpu
->kvm
->lock
);
484 * Does the new cr3 value map to physical memory? (Note, we
485 * catch an invalid cr3 even in real-mode, because it would
486 * cause trouble later on when we turn on paging anyway.)
488 * A real CPU would silently accept an invalid cr3 and would
489 * attempt to use it - with largely undefined (and often hard
490 * to debug) behavior on the guest side.
492 if (unlikely(!gfn_to_memslot(vcpu
->kvm
, cr3
>> PAGE_SHIFT
)))
495 vcpu
->mmu
.new_cr3(vcpu
);
496 spin_unlock(&vcpu
->kvm
->lock
);
498 EXPORT_SYMBOL_GPL(set_cr3
);
500 void set_cr8(struct kvm_vcpu
*vcpu
, unsigned long cr8
)
502 if ( cr8
& CR8_RESEVED_BITS
) {
503 printk(KERN_DEBUG
"set_cr8: #GP, reserved bits 0x%lx\n", cr8
);
509 EXPORT_SYMBOL_GPL(set_cr8
);
511 void fx_init(struct kvm_vcpu
*vcpu
)
513 struct __attribute__ ((__packed__
)) fx_image_s
{
519 u64 operand
;// fpu dp
525 fx_save(vcpu
->host_fx_image
);
527 fx_save(vcpu
->guest_fx_image
);
528 fx_restore(vcpu
->host_fx_image
);
530 fx_image
= (struct fx_image_s
*)vcpu
->guest_fx_image
;
531 fx_image
->mxcsr
= 0x1f80;
532 memset(vcpu
->guest_fx_image
+ sizeof(struct fx_image_s
),
533 0, FX_IMAGE_SIZE
- sizeof(struct fx_image_s
));
535 EXPORT_SYMBOL_GPL(fx_init
);
538 * Creates some virtual cpus. Good luck creating more than one.
540 static int kvm_dev_ioctl_create_vcpu(struct kvm
*kvm
, int n
)
543 struct kvm_vcpu
*vcpu
;
549 vcpu
= &kvm
->vcpus
[n
];
551 mutex_lock(&vcpu
->mutex
);
554 mutex_unlock(&vcpu
->mutex
);
558 vcpu
->host_fx_image
= (char*)ALIGN((hva_t
)vcpu
->fx_buf
,
560 vcpu
->guest_fx_image
= vcpu
->host_fx_image
+ FX_IMAGE_SIZE
;
562 r
= kvm_arch_ops
->vcpu_create(vcpu
);
566 r
= kvm_mmu_create(vcpu
);
570 kvm_arch_ops
->vcpu_load(vcpu
);
571 r
= kvm_mmu_setup(vcpu
);
573 r
= kvm_arch_ops
->vcpu_setup(vcpu
);
583 mutex_unlock(&vcpu
->mutex
);
589 * Allocate some memory and give it an address in the guest physical address
592 * Discontiguous memory is allowed, mostly for framebuffers.
594 static int kvm_dev_ioctl_set_memory_region(struct kvm
*kvm
,
595 struct kvm_memory_region
*mem
)
599 unsigned long npages
;
601 struct kvm_memory_slot
*memslot
;
602 struct kvm_memory_slot old
, new;
603 int memory_config_version
;
606 /* General sanity checks */
607 if (mem
->memory_size
& (PAGE_SIZE
- 1))
609 if (mem
->guest_phys_addr
& (PAGE_SIZE
- 1))
611 if (mem
->slot
>= KVM_MEMORY_SLOTS
)
613 if (mem
->guest_phys_addr
+ mem
->memory_size
< mem
->guest_phys_addr
)
616 memslot
= &kvm
->memslots
[mem
->slot
];
617 base_gfn
= mem
->guest_phys_addr
>> PAGE_SHIFT
;
618 npages
= mem
->memory_size
>> PAGE_SHIFT
;
621 mem
->flags
&= ~KVM_MEM_LOG_DIRTY_PAGES
;
624 spin_lock(&kvm
->lock
);
626 memory_config_version
= kvm
->memory_config_version
;
627 new = old
= *memslot
;
629 new.base_gfn
= base_gfn
;
631 new.flags
= mem
->flags
;
633 /* Disallow changing a memory slot's size. */
635 if (npages
&& old
.npages
&& npages
!= old
.npages
)
638 /* Check for overlaps */
640 for (i
= 0; i
< KVM_MEMORY_SLOTS
; ++i
) {
641 struct kvm_memory_slot
*s
= &kvm
->memslots
[i
];
645 if (!((base_gfn
+ npages
<= s
->base_gfn
) ||
646 (base_gfn
>= s
->base_gfn
+ s
->npages
)))
650 * Do memory allocations outside lock. memory_config_version will
653 spin_unlock(&kvm
->lock
);
655 /* Deallocate if slot is being removed */
659 /* Free page dirty bitmap if unneeded */
660 if (!(new.flags
& KVM_MEM_LOG_DIRTY_PAGES
))
661 new.dirty_bitmap
= NULL
;
665 /* Allocate if a slot is being created */
666 if (npages
&& !new.phys_mem
) {
667 new.phys_mem
= vmalloc(npages
* sizeof(struct page
*));
672 memset(new.phys_mem
, 0, npages
* sizeof(struct page
*));
673 for (i
= 0; i
< npages
; ++i
) {
674 new.phys_mem
[i
] = alloc_page(GFP_HIGHUSER
676 if (!new.phys_mem
[i
])
678 set_page_private(new.phys_mem
[i
],0);
682 /* Allocate page dirty bitmap if needed */
683 if ((new.flags
& KVM_MEM_LOG_DIRTY_PAGES
) && !new.dirty_bitmap
) {
684 unsigned dirty_bytes
= ALIGN(npages
, BITS_PER_LONG
) / 8;
686 new.dirty_bitmap
= vmalloc(dirty_bytes
);
687 if (!new.dirty_bitmap
)
689 memset(new.dirty_bitmap
, 0, dirty_bytes
);
692 spin_lock(&kvm
->lock
);
694 if (memory_config_version
!= kvm
->memory_config_version
) {
695 spin_unlock(&kvm
->lock
);
696 kvm_free_physmem_slot(&new, &old
);
704 if (mem
->slot
>= kvm
->nmemslots
)
705 kvm
->nmemslots
= mem
->slot
+ 1;
708 ++kvm
->memory_config_version
;
710 spin_unlock(&kvm
->lock
);
712 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
713 struct kvm_vcpu
*vcpu
;
715 vcpu
= vcpu_load(kvm
, i
);
718 kvm_mmu_reset_context(vcpu
);
722 kvm_free_physmem_slot(&old
, &new);
726 spin_unlock(&kvm
->lock
);
728 kvm_free_physmem_slot(&new, &old
);
733 static void do_remove_write_access(struct kvm_vcpu
*vcpu
, int slot
)
735 spin_lock(&vcpu
->kvm
->lock
);
736 kvm_mmu_slot_remove_write_access(vcpu
, slot
);
737 spin_unlock(&vcpu
->kvm
->lock
);
741 * Get (and clear) the dirty memory log for a memory slot.
743 static int kvm_dev_ioctl_get_dirty_log(struct kvm
*kvm
,
744 struct kvm_dirty_log
*log
)
746 struct kvm_memory_slot
*memslot
;
750 unsigned long any
= 0;
752 spin_lock(&kvm
->lock
);
755 * Prevent changes to guest memory configuration even while the lock
759 spin_unlock(&kvm
->lock
);
761 if (log
->slot
>= KVM_MEMORY_SLOTS
)
764 memslot
= &kvm
->memslots
[log
->slot
];
766 if (!memslot
->dirty_bitmap
)
769 n
= ALIGN(memslot
->npages
, 8) / 8;
771 for (i
= 0; !any
&& i
< n
; ++i
)
772 any
= memslot
->dirty_bitmap
[i
];
775 if (copy_to_user(log
->dirty_bitmap
, memslot
->dirty_bitmap
, n
))
780 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
781 struct kvm_vcpu
*vcpu
= vcpu_load(kvm
, i
);
786 do_remove_write_access(vcpu
, log
->slot
);
787 memset(memslot
->dirty_bitmap
, 0, n
);
790 kvm_arch_ops
->tlb_flush(vcpu
);
798 spin_lock(&kvm
->lock
);
800 spin_unlock(&kvm
->lock
);
804 struct kvm_memory_slot
*gfn_to_memslot(struct kvm
*kvm
, gfn_t gfn
)
808 for (i
= 0; i
< kvm
->nmemslots
; ++i
) {
809 struct kvm_memory_slot
*memslot
= &kvm
->memslots
[i
];
811 if (gfn
>= memslot
->base_gfn
812 && gfn
< memslot
->base_gfn
+ memslot
->npages
)
817 EXPORT_SYMBOL_GPL(gfn_to_memslot
);
819 void mark_page_dirty(struct kvm
*kvm
, gfn_t gfn
)
822 struct kvm_memory_slot
*memslot
= NULL
;
823 unsigned long rel_gfn
;
825 for (i
= 0; i
< kvm
->nmemslots
; ++i
) {
826 memslot
= &kvm
->memslots
[i
];
828 if (gfn
>= memslot
->base_gfn
829 && gfn
< memslot
->base_gfn
+ memslot
->npages
) {
831 if (!memslot
|| !memslot
->dirty_bitmap
)
834 rel_gfn
= gfn
- memslot
->base_gfn
;
837 if (!test_bit(rel_gfn
, memslot
->dirty_bitmap
))
838 set_bit(rel_gfn
, memslot
->dirty_bitmap
);
844 static int emulator_read_std(unsigned long addr
,
847 struct x86_emulate_ctxt
*ctxt
)
849 struct kvm_vcpu
*vcpu
= ctxt
->vcpu
;
853 gpa_t gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, addr
);
854 unsigned offset
= addr
& (PAGE_SIZE
-1);
855 unsigned tocopy
= min(bytes
, (unsigned)PAGE_SIZE
- offset
);
857 struct kvm_memory_slot
*memslot
;
860 if (gpa
== UNMAPPED_GVA
)
861 return X86EMUL_PROPAGATE_FAULT
;
862 pfn
= gpa
>> PAGE_SHIFT
;
863 memslot
= gfn_to_memslot(vcpu
->kvm
, pfn
);
865 return X86EMUL_UNHANDLEABLE
;
866 page
= kmap_atomic(gfn_to_page(memslot
, pfn
), KM_USER0
);
868 memcpy(data
, page
+ offset
, tocopy
);
870 kunmap_atomic(page
, KM_USER0
);
877 return X86EMUL_CONTINUE
;
880 static int emulator_write_std(unsigned long addr
,
883 struct x86_emulate_ctxt
*ctxt
)
885 printk(KERN_ERR
"emulator_write_std: addr %lx n %d\n",
887 return X86EMUL_UNHANDLEABLE
;
890 static int emulator_read_emulated(unsigned long addr
,
893 struct x86_emulate_ctxt
*ctxt
)
895 struct kvm_vcpu
*vcpu
= ctxt
->vcpu
;
897 if (vcpu
->mmio_read_completed
) {
898 memcpy(val
, vcpu
->mmio_data
, bytes
);
899 vcpu
->mmio_read_completed
= 0;
900 return X86EMUL_CONTINUE
;
901 } else if (emulator_read_std(addr
, val
, bytes
, ctxt
)
903 return X86EMUL_CONTINUE
;
905 gpa_t gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, addr
);
907 if (gpa
== UNMAPPED_GVA
)
908 return X86EMUL_PROPAGATE_FAULT
;
909 vcpu
->mmio_needed
= 1;
910 vcpu
->mmio_phys_addr
= gpa
;
911 vcpu
->mmio_size
= bytes
;
912 vcpu
->mmio_is_write
= 0;
914 return X86EMUL_UNHANDLEABLE
;
918 static int emulator_write_phys(struct kvm_vcpu
*vcpu
, gpa_t gpa
,
919 unsigned long val
, int bytes
)
921 struct kvm_memory_slot
*m
;
925 if (((gpa
+ bytes
- 1) >> PAGE_SHIFT
) != (gpa
>> PAGE_SHIFT
))
927 m
= gfn_to_memslot(vcpu
->kvm
, gpa
>> PAGE_SHIFT
);
930 page
= gfn_to_page(m
, gpa
>> PAGE_SHIFT
);
931 kvm_mmu_pre_write(vcpu
, gpa
, bytes
);
932 virt
= kmap_atomic(page
, KM_USER0
);
933 memcpy(virt
+ offset_in_page(gpa
), &val
, bytes
);
934 kunmap_atomic(virt
, KM_USER0
);
935 kvm_mmu_post_write(vcpu
, gpa
, bytes
);
939 static int emulator_write_emulated(unsigned long addr
,
942 struct x86_emulate_ctxt
*ctxt
)
944 struct kvm_vcpu
*vcpu
= ctxt
->vcpu
;
945 gpa_t gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, addr
);
947 if (gpa
== UNMAPPED_GVA
)
948 return X86EMUL_PROPAGATE_FAULT
;
950 if (emulator_write_phys(vcpu
, gpa
, val
, bytes
))
951 return X86EMUL_CONTINUE
;
953 vcpu
->mmio_needed
= 1;
954 vcpu
->mmio_phys_addr
= gpa
;
955 vcpu
->mmio_size
= bytes
;
956 vcpu
->mmio_is_write
= 1;
957 memcpy(vcpu
->mmio_data
, &val
, bytes
);
959 return X86EMUL_CONTINUE
;
962 static int emulator_cmpxchg_emulated(unsigned long addr
,
966 struct x86_emulate_ctxt
*ctxt
)
972 printk(KERN_WARNING
"kvm: emulating exchange as write\n");
974 return emulator_write_emulated(addr
, new, bytes
, ctxt
);
979 static int emulator_cmpxchg8b_emulated(unsigned long addr
,
980 unsigned long old_lo
,
981 unsigned long old_hi
,
982 unsigned long new_lo
,
983 unsigned long new_hi
,
984 struct x86_emulate_ctxt
*ctxt
)
991 printk(KERN_WARNING
"kvm: emulating exchange8b as write\n");
993 r
= emulator_write_emulated(addr
, new_lo
, 4, ctxt
);
994 if (r
!= X86EMUL_CONTINUE
)
996 return emulator_write_emulated(addr
+4, new_hi
, 4, ctxt
);
1001 static unsigned long get_segment_base(struct kvm_vcpu
*vcpu
, int seg
)
1003 return kvm_arch_ops
->get_segment_base(vcpu
, seg
);
1006 int emulate_invlpg(struct kvm_vcpu
*vcpu
, gva_t address
)
1008 return X86EMUL_CONTINUE
;
1011 int emulate_clts(struct kvm_vcpu
*vcpu
)
1015 kvm_arch_ops
->decache_cr0_cr4_guest_bits(vcpu
);
1016 cr0
= vcpu
->cr0
& ~CR0_TS_MASK
;
1017 kvm_arch_ops
->set_cr0(vcpu
, cr0
);
1018 return X86EMUL_CONTINUE
;
1021 int emulator_get_dr(struct x86_emulate_ctxt
* ctxt
, int dr
, unsigned long *dest
)
1023 struct kvm_vcpu
*vcpu
= ctxt
->vcpu
;
1027 *dest
= kvm_arch_ops
->get_dr(vcpu
, dr
);
1028 return X86EMUL_CONTINUE
;
1030 printk(KERN_DEBUG
"%s: unexpected dr %u\n",
1032 return X86EMUL_UNHANDLEABLE
;
1036 int emulator_set_dr(struct x86_emulate_ctxt
*ctxt
, int dr
, unsigned long value
)
1038 unsigned long mask
= (ctxt
->mode
== X86EMUL_MODE_PROT64
) ? ~0ULL : ~0U;
1041 kvm_arch_ops
->set_dr(ctxt
->vcpu
, dr
, value
& mask
, &exception
);
1043 /* FIXME: better handling */
1044 return X86EMUL_UNHANDLEABLE
;
1046 return X86EMUL_CONTINUE
;
1049 static void report_emulation_failure(struct x86_emulate_ctxt
*ctxt
)
1051 static int reported
;
1053 unsigned long rip
= ctxt
->vcpu
->rip
;
1054 unsigned long rip_linear
;
1056 rip_linear
= rip
+ get_segment_base(ctxt
->vcpu
, VCPU_SREG_CS
);
1061 emulator_read_std(rip_linear
, (void *)opcodes
, 4, ctxt
);
1063 printk(KERN_ERR
"emulation failed but !mmio_needed?"
1064 " rip %lx %02x %02x %02x %02x\n",
1065 rip
, opcodes
[0], opcodes
[1], opcodes
[2], opcodes
[3]);
1069 struct x86_emulate_ops emulate_ops
= {
1070 .read_std
= emulator_read_std
,
1071 .write_std
= emulator_write_std
,
1072 .read_emulated
= emulator_read_emulated
,
1073 .write_emulated
= emulator_write_emulated
,
1074 .cmpxchg_emulated
= emulator_cmpxchg_emulated
,
1075 #ifdef CONFIG_X86_32
1076 .cmpxchg8b_emulated
= emulator_cmpxchg8b_emulated
,
1080 int emulate_instruction(struct kvm_vcpu
*vcpu
,
1081 struct kvm_run
*run
,
1085 struct x86_emulate_ctxt emulate_ctxt
;
1089 kvm_arch_ops
->cache_regs(vcpu
);
1091 kvm_arch_ops
->get_cs_db_l_bits(vcpu
, &cs_db
, &cs_l
);
1093 emulate_ctxt
.vcpu
= vcpu
;
1094 emulate_ctxt
.eflags
= kvm_arch_ops
->get_rflags(vcpu
);
1095 emulate_ctxt
.cr2
= cr2
;
1096 emulate_ctxt
.mode
= (emulate_ctxt
.eflags
& X86_EFLAGS_VM
)
1097 ? X86EMUL_MODE_REAL
: cs_l
1098 ? X86EMUL_MODE_PROT64
: cs_db
1099 ? X86EMUL_MODE_PROT32
: X86EMUL_MODE_PROT16
;
1101 if (emulate_ctxt
.mode
== X86EMUL_MODE_PROT64
) {
1102 emulate_ctxt
.cs_base
= 0;
1103 emulate_ctxt
.ds_base
= 0;
1104 emulate_ctxt
.es_base
= 0;
1105 emulate_ctxt
.ss_base
= 0;
1107 emulate_ctxt
.cs_base
= get_segment_base(vcpu
, VCPU_SREG_CS
);
1108 emulate_ctxt
.ds_base
= get_segment_base(vcpu
, VCPU_SREG_DS
);
1109 emulate_ctxt
.es_base
= get_segment_base(vcpu
, VCPU_SREG_ES
);
1110 emulate_ctxt
.ss_base
= get_segment_base(vcpu
, VCPU_SREG_SS
);
1113 emulate_ctxt
.gs_base
= get_segment_base(vcpu
, VCPU_SREG_GS
);
1114 emulate_ctxt
.fs_base
= get_segment_base(vcpu
, VCPU_SREG_FS
);
1116 vcpu
->mmio_is_write
= 0;
1117 r
= x86_emulate_memop(&emulate_ctxt
, &emulate_ops
);
1119 if ((r
|| vcpu
->mmio_is_write
) && run
) {
1120 run
->mmio
.phys_addr
= vcpu
->mmio_phys_addr
;
1121 memcpy(run
->mmio
.data
, vcpu
->mmio_data
, 8);
1122 run
->mmio
.len
= vcpu
->mmio_size
;
1123 run
->mmio
.is_write
= vcpu
->mmio_is_write
;
1127 if (kvm_mmu_unprotect_page_virt(vcpu
, cr2
))
1128 return EMULATE_DONE
;
1129 if (!vcpu
->mmio_needed
) {
1130 report_emulation_failure(&emulate_ctxt
);
1131 return EMULATE_FAIL
;
1133 return EMULATE_DO_MMIO
;
1136 kvm_arch_ops
->decache_regs(vcpu
);
1137 kvm_arch_ops
->set_rflags(vcpu
, emulate_ctxt
.eflags
);
1139 if (vcpu
->mmio_is_write
)
1140 return EMULATE_DO_MMIO
;
1142 return EMULATE_DONE
;
1144 EXPORT_SYMBOL_GPL(emulate_instruction
);
1146 int kvm_hypercall(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
)
1148 unsigned long nr
, a0
, a1
, a2
, a3
, a4
, a5
, ret
;
1150 kvm_arch_ops
->decache_regs(vcpu
);
1152 #ifdef CONFIG_X86_64
1153 if (is_long_mode(vcpu
)) {
1154 nr
= vcpu
->regs
[VCPU_REGS_RAX
];
1155 a0
= vcpu
->regs
[VCPU_REGS_RDI
];
1156 a1
= vcpu
->regs
[VCPU_REGS_RSI
];
1157 a2
= vcpu
->regs
[VCPU_REGS_RDX
];
1158 a3
= vcpu
->regs
[VCPU_REGS_RCX
];
1159 a4
= vcpu
->regs
[VCPU_REGS_R8
];
1160 a5
= vcpu
->regs
[VCPU_REGS_R9
];
1164 nr
= vcpu
->regs
[VCPU_REGS_RBX
] & -1u;
1165 a0
= vcpu
->regs
[VCPU_REGS_RAX
] & -1u;
1166 a1
= vcpu
->regs
[VCPU_REGS_RCX
] & -1u;
1167 a2
= vcpu
->regs
[VCPU_REGS_RDX
] & -1u;
1168 a3
= vcpu
->regs
[VCPU_REGS_RSI
] & -1u;
1169 a4
= vcpu
->regs
[VCPU_REGS_RDI
] & -1u;
1170 a5
= vcpu
->regs
[VCPU_REGS_RBP
] & -1u;
1176 vcpu
->regs
[VCPU_REGS_RAX
] = ret
;
1177 kvm_arch_ops
->cache_regs(vcpu
);
1180 EXPORT_SYMBOL_GPL(kvm_hypercall
);
1182 static u64
mk_cr_64(u64 curr_cr
, u32 new_val
)
1184 return (curr_cr
& ~((1ULL << 32) - 1)) | new_val
;
1187 void realmode_lgdt(struct kvm_vcpu
*vcpu
, u16 limit
, unsigned long base
)
1189 struct descriptor_table dt
= { limit
, base
};
1191 kvm_arch_ops
->set_gdt(vcpu
, &dt
);
1194 void realmode_lidt(struct kvm_vcpu
*vcpu
, u16 limit
, unsigned long base
)
1196 struct descriptor_table dt
= { limit
, base
};
1198 kvm_arch_ops
->set_idt(vcpu
, &dt
);
1201 void realmode_lmsw(struct kvm_vcpu
*vcpu
, unsigned long msw
,
1202 unsigned long *rflags
)
1205 *rflags
= kvm_arch_ops
->get_rflags(vcpu
);
1208 unsigned long realmode_get_cr(struct kvm_vcpu
*vcpu
, int cr
)
1210 kvm_arch_ops
->decache_cr0_cr4_guest_bits(vcpu
);
1221 vcpu_printf(vcpu
, "%s: unexpected cr %u\n", __FUNCTION__
, cr
);
1226 void realmode_set_cr(struct kvm_vcpu
*vcpu
, int cr
, unsigned long val
,
1227 unsigned long *rflags
)
1231 set_cr0(vcpu
, mk_cr_64(vcpu
->cr0
, val
));
1232 *rflags
= kvm_arch_ops
->get_rflags(vcpu
);
1241 set_cr4(vcpu
, mk_cr_64(vcpu
->cr4
, val
));
1244 vcpu_printf(vcpu
, "%s: unexpected cr %u\n", __FUNCTION__
, cr
);
1249 * Register the para guest with the host:
1251 static int vcpu_register_para(struct kvm_vcpu
*vcpu
, gpa_t para_state_gpa
)
1253 struct kvm_vcpu_para_state
*para_state
;
1254 hpa_t para_state_hpa
, hypercall_hpa
;
1255 struct page
*para_state_page
;
1256 unsigned char *hypercall
;
1257 gpa_t hypercall_gpa
;
1259 printk(KERN_DEBUG
"kvm: guest trying to enter paravirtual mode\n");
1260 printk(KERN_DEBUG
".... para_state_gpa: %08Lx\n", para_state_gpa
);
1263 * Needs to be page aligned:
1265 if (para_state_gpa
!= PAGE_ALIGN(para_state_gpa
))
1268 para_state_hpa
= gpa_to_hpa(vcpu
, para_state_gpa
);
1269 printk(KERN_DEBUG
".... para_state_hpa: %08Lx\n", para_state_hpa
);
1270 if (is_error_hpa(para_state_hpa
))
1273 para_state_page
= pfn_to_page(para_state_hpa
>> PAGE_SHIFT
);
1274 para_state
= kmap_atomic(para_state_page
, KM_USER0
);
1276 printk(KERN_DEBUG
".... guest version: %d\n", para_state
->guest_version
);
1277 printk(KERN_DEBUG
".... size: %d\n", para_state
->size
);
1279 para_state
->host_version
= KVM_PARA_API_VERSION
;
1281 * We cannot support guests that try to register themselves
1282 * with a newer API version than the host supports:
1284 if (para_state
->guest_version
> KVM_PARA_API_VERSION
) {
1285 para_state
->ret
= -KVM_EINVAL
;
1286 goto err_kunmap_skip
;
1289 hypercall_gpa
= para_state
->hypercall_gpa
;
1290 hypercall_hpa
= gpa_to_hpa(vcpu
, hypercall_gpa
);
1291 printk(KERN_DEBUG
".... hypercall_hpa: %08Lx\n", hypercall_hpa
);
1292 if (is_error_hpa(hypercall_hpa
)) {
1293 para_state
->ret
= -KVM_EINVAL
;
1294 goto err_kunmap_skip
;
1297 printk(KERN_DEBUG
"kvm: para guest successfully registered.\n");
1298 vcpu
->para_state_page
= para_state_page
;
1299 vcpu
->para_state_gpa
= para_state_gpa
;
1300 vcpu
->hypercall_gpa
= hypercall_gpa
;
1302 hypercall
= kmap_atomic(pfn_to_page(hypercall_hpa
>> PAGE_SHIFT
),
1303 KM_USER1
) + (hypercall_hpa
& ~PAGE_MASK
);
1304 kvm_arch_ops
->patch_hypercall(vcpu
, hypercall
);
1305 kunmap_atomic(hypercall
, KM_USER1
);
1307 para_state
->ret
= 0;
1309 kunmap_atomic(para_state
, KM_USER0
);
1315 int kvm_get_msr_common(struct kvm_vcpu
*vcpu
, u32 msr
, u64
*pdata
)
1320 case 0xc0010010: /* SYSCFG */
1321 case 0xc0010015: /* HWCR */
1322 case MSR_IA32_PLATFORM_ID
:
1323 case MSR_IA32_P5_MC_ADDR
:
1324 case MSR_IA32_P5_MC_TYPE
:
1325 case MSR_IA32_MC0_CTL
:
1326 case MSR_IA32_MCG_STATUS
:
1327 case MSR_IA32_MCG_CAP
:
1328 case MSR_IA32_MC0_MISC
:
1329 case MSR_IA32_MC0_MISC
+4:
1330 case MSR_IA32_MC0_MISC
+8:
1331 case MSR_IA32_MC0_MISC
+12:
1332 case MSR_IA32_MC0_MISC
+16:
1333 case MSR_IA32_UCODE_REV
:
1334 case MSR_IA32_PERF_STATUS
:
1335 /* MTRR registers */
1337 case 0x200 ... 0x2ff:
1340 case 0xcd: /* fsb frequency */
1343 case MSR_IA32_APICBASE
:
1344 data
= vcpu
->apic_base
;
1346 case MSR_IA32_MISC_ENABLE
:
1347 data
= vcpu
->ia32_misc_enable_msr
;
1349 #ifdef CONFIG_X86_64
1351 data
= vcpu
->shadow_efer
;
1355 printk(KERN_ERR
"kvm: unhandled rdmsr: 0x%x\n", msr
);
1361 EXPORT_SYMBOL_GPL(kvm_get_msr_common
);
1364 * Reads an msr value (of 'msr_index') into 'pdata'.
1365 * Returns 0 on success, non-0 otherwise.
1366 * Assumes vcpu_load() was already called.
1368 static int get_msr(struct kvm_vcpu
*vcpu
, u32 msr_index
, u64
*pdata
)
1370 return kvm_arch_ops
->get_msr(vcpu
, msr_index
, pdata
);
1373 #ifdef CONFIG_X86_64
1375 static void set_efer(struct kvm_vcpu
*vcpu
, u64 efer
)
1377 if (efer
& EFER_RESERVED_BITS
) {
1378 printk(KERN_DEBUG
"set_efer: 0x%llx #GP, reserved bits\n",
1385 && (vcpu
->shadow_efer
& EFER_LME
) != (efer
& EFER_LME
)) {
1386 printk(KERN_DEBUG
"set_efer: #GP, change LME while paging\n");
1391 kvm_arch_ops
->set_efer(vcpu
, efer
);
1394 efer
|= vcpu
->shadow_efer
& EFER_LMA
;
1396 vcpu
->shadow_efer
= efer
;
1401 int kvm_set_msr_common(struct kvm_vcpu
*vcpu
, u32 msr
, u64 data
)
1404 #ifdef CONFIG_X86_64
1406 set_efer(vcpu
, data
);
1409 case MSR_IA32_MC0_STATUS
:
1410 printk(KERN_WARNING
"%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1411 __FUNCTION__
, data
);
1413 case MSR_IA32_UCODE_REV
:
1414 case MSR_IA32_UCODE_WRITE
:
1415 case 0x200 ... 0x2ff: /* MTRRs */
1417 case MSR_IA32_APICBASE
:
1418 vcpu
->apic_base
= data
;
1420 case MSR_IA32_MISC_ENABLE
:
1421 vcpu
->ia32_misc_enable_msr
= data
;
1424 * This is the 'probe whether the host is KVM' logic:
1426 case MSR_KVM_API_MAGIC
:
1427 return vcpu_register_para(vcpu
, data
);
1430 printk(KERN_ERR
"kvm: unhandled wrmsr: 0x%x\n", msr
);
1435 EXPORT_SYMBOL_GPL(kvm_set_msr_common
);
1438 * Writes msr value into into the appropriate "register".
1439 * Returns 0 on success, non-0 otherwise.
1440 * Assumes vcpu_load() was already called.
1442 static int set_msr(struct kvm_vcpu
*vcpu
, u32 msr_index
, u64 data
)
1444 return kvm_arch_ops
->set_msr(vcpu
, msr_index
, data
);
1447 void kvm_resched(struct kvm_vcpu
*vcpu
)
1451 /* Cannot fail - no vcpu unplug yet. */
1452 vcpu_load(vcpu
->kvm
, vcpu_slot(vcpu
));
1454 EXPORT_SYMBOL_GPL(kvm_resched
);
1456 void load_msrs(struct vmx_msr_entry
*e
, int n
)
1460 for (i
= 0; i
< n
; ++i
)
1461 wrmsrl(e
[i
].index
, e
[i
].data
);
1463 EXPORT_SYMBOL_GPL(load_msrs
);
1465 void save_msrs(struct vmx_msr_entry
*e
, int n
)
1469 for (i
= 0; i
< n
; ++i
)
1470 rdmsrl(e
[i
].index
, e
[i
].data
);
1472 EXPORT_SYMBOL_GPL(save_msrs
);
1474 static int kvm_dev_ioctl_run(struct kvm
*kvm
, struct kvm_run
*kvm_run
)
1476 struct kvm_vcpu
*vcpu
;
1479 if (!valid_vcpu(kvm_run
->vcpu
))
1482 vcpu
= vcpu_load(kvm
, kvm_run
->vcpu
);
1486 /* re-sync apic's tpr */
1487 vcpu
->cr8
= kvm_run
->cr8
;
1489 if (kvm_run
->emulated
) {
1490 kvm_arch_ops
->skip_emulated_instruction(vcpu
);
1491 kvm_run
->emulated
= 0;
1494 if (kvm_run
->mmio_completed
) {
1495 memcpy(vcpu
->mmio_data
, kvm_run
->mmio
.data
, 8);
1496 vcpu
->mmio_read_completed
= 1;
1499 vcpu
->mmio_needed
= 0;
1501 r
= kvm_arch_ops
->run(vcpu
, kvm_run
);
1507 static int kvm_dev_ioctl_get_regs(struct kvm
*kvm
, struct kvm_regs
*regs
)
1509 struct kvm_vcpu
*vcpu
;
1511 if (!valid_vcpu(regs
->vcpu
))
1514 vcpu
= vcpu_load(kvm
, regs
->vcpu
);
1518 kvm_arch_ops
->cache_regs(vcpu
);
1520 regs
->rax
= vcpu
->regs
[VCPU_REGS_RAX
];
1521 regs
->rbx
= vcpu
->regs
[VCPU_REGS_RBX
];
1522 regs
->rcx
= vcpu
->regs
[VCPU_REGS_RCX
];
1523 regs
->rdx
= vcpu
->regs
[VCPU_REGS_RDX
];
1524 regs
->rsi
= vcpu
->regs
[VCPU_REGS_RSI
];
1525 regs
->rdi
= vcpu
->regs
[VCPU_REGS_RDI
];
1526 regs
->rsp
= vcpu
->regs
[VCPU_REGS_RSP
];
1527 regs
->rbp
= vcpu
->regs
[VCPU_REGS_RBP
];
1528 #ifdef CONFIG_X86_64
1529 regs
->r8
= vcpu
->regs
[VCPU_REGS_R8
];
1530 regs
->r9
= vcpu
->regs
[VCPU_REGS_R9
];
1531 regs
->r10
= vcpu
->regs
[VCPU_REGS_R10
];
1532 regs
->r11
= vcpu
->regs
[VCPU_REGS_R11
];
1533 regs
->r12
= vcpu
->regs
[VCPU_REGS_R12
];
1534 regs
->r13
= vcpu
->regs
[VCPU_REGS_R13
];
1535 regs
->r14
= vcpu
->regs
[VCPU_REGS_R14
];
1536 regs
->r15
= vcpu
->regs
[VCPU_REGS_R15
];
1539 regs
->rip
= vcpu
->rip
;
1540 regs
->rflags
= kvm_arch_ops
->get_rflags(vcpu
);
1543 * Don't leak debug flags in case they were set for guest debugging
1545 if (vcpu
->guest_debug
.enabled
&& vcpu
->guest_debug
.singlestep
)
1546 regs
->rflags
&= ~(X86_EFLAGS_TF
| X86_EFLAGS_RF
);
1553 static int kvm_dev_ioctl_set_regs(struct kvm
*kvm
, struct kvm_regs
*regs
)
1555 struct kvm_vcpu
*vcpu
;
1557 if (!valid_vcpu(regs
->vcpu
))
1560 vcpu
= vcpu_load(kvm
, regs
->vcpu
);
1564 vcpu
->regs
[VCPU_REGS_RAX
] = regs
->rax
;
1565 vcpu
->regs
[VCPU_REGS_RBX
] = regs
->rbx
;
1566 vcpu
->regs
[VCPU_REGS_RCX
] = regs
->rcx
;
1567 vcpu
->regs
[VCPU_REGS_RDX
] = regs
->rdx
;
1568 vcpu
->regs
[VCPU_REGS_RSI
] = regs
->rsi
;
1569 vcpu
->regs
[VCPU_REGS_RDI
] = regs
->rdi
;
1570 vcpu
->regs
[VCPU_REGS_RSP
] = regs
->rsp
;
1571 vcpu
->regs
[VCPU_REGS_RBP
] = regs
->rbp
;
1572 #ifdef CONFIG_X86_64
1573 vcpu
->regs
[VCPU_REGS_R8
] = regs
->r8
;
1574 vcpu
->regs
[VCPU_REGS_R9
] = regs
->r9
;
1575 vcpu
->regs
[VCPU_REGS_R10
] = regs
->r10
;
1576 vcpu
->regs
[VCPU_REGS_R11
] = regs
->r11
;
1577 vcpu
->regs
[VCPU_REGS_R12
] = regs
->r12
;
1578 vcpu
->regs
[VCPU_REGS_R13
] = regs
->r13
;
1579 vcpu
->regs
[VCPU_REGS_R14
] = regs
->r14
;
1580 vcpu
->regs
[VCPU_REGS_R15
] = regs
->r15
;
1583 vcpu
->rip
= regs
->rip
;
1584 kvm_arch_ops
->set_rflags(vcpu
, regs
->rflags
);
1586 kvm_arch_ops
->decache_regs(vcpu
);
1593 static void get_segment(struct kvm_vcpu
*vcpu
,
1594 struct kvm_segment
*var
, int seg
)
1596 return kvm_arch_ops
->get_segment(vcpu
, var
, seg
);
1599 static int kvm_dev_ioctl_get_sregs(struct kvm
*kvm
, struct kvm_sregs
*sregs
)
1601 struct kvm_vcpu
*vcpu
;
1602 struct descriptor_table dt
;
1604 if (!valid_vcpu(sregs
->vcpu
))
1606 vcpu
= vcpu_load(kvm
, sregs
->vcpu
);
1610 get_segment(vcpu
, &sregs
->cs
, VCPU_SREG_CS
);
1611 get_segment(vcpu
, &sregs
->ds
, VCPU_SREG_DS
);
1612 get_segment(vcpu
, &sregs
->es
, VCPU_SREG_ES
);
1613 get_segment(vcpu
, &sregs
->fs
, VCPU_SREG_FS
);
1614 get_segment(vcpu
, &sregs
->gs
, VCPU_SREG_GS
);
1615 get_segment(vcpu
, &sregs
->ss
, VCPU_SREG_SS
);
1617 get_segment(vcpu
, &sregs
->tr
, VCPU_SREG_TR
);
1618 get_segment(vcpu
, &sregs
->ldt
, VCPU_SREG_LDTR
);
1620 kvm_arch_ops
->get_idt(vcpu
, &dt
);
1621 sregs
->idt
.limit
= dt
.limit
;
1622 sregs
->idt
.base
= dt
.base
;
1623 kvm_arch_ops
->get_gdt(vcpu
, &dt
);
1624 sregs
->gdt
.limit
= dt
.limit
;
1625 sregs
->gdt
.base
= dt
.base
;
1627 kvm_arch_ops
->decache_cr0_cr4_guest_bits(vcpu
);
1628 sregs
->cr0
= vcpu
->cr0
;
1629 sregs
->cr2
= vcpu
->cr2
;
1630 sregs
->cr3
= vcpu
->cr3
;
1631 sregs
->cr4
= vcpu
->cr4
;
1632 sregs
->cr8
= vcpu
->cr8
;
1633 sregs
->efer
= vcpu
->shadow_efer
;
1634 sregs
->apic_base
= vcpu
->apic_base
;
1636 memcpy(sregs
->interrupt_bitmap
, vcpu
->irq_pending
,
1637 sizeof sregs
->interrupt_bitmap
);
1644 static void set_segment(struct kvm_vcpu
*vcpu
,
1645 struct kvm_segment
*var
, int seg
)
1647 return kvm_arch_ops
->set_segment(vcpu
, var
, seg
);
1650 static int kvm_dev_ioctl_set_sregs(struct kvm
*kvm
, struct kvm_sregs
*sregs
)
1652 struct kvm_vcpu
*vcpu
;
1653 int mmu_reset_needed
= 0;
1655 struct descriptor_table dt
;
1657 if (!valid_vcpu(sregs
->vcpu
))
1659 vcpu
= vcpu_load(kvm
, sregs
->vcpu
);
1663 set_segment(vcpu
, &sregs
->cs
, VCPU_SREG_CS
);
1664 set_segment(vcpu
, &sregs
->ds
, VCPU_SREG_DS
);
1665 set_segment(vcpu
, &sregs
->es
, VCPU_SREG_ES
);
1666 set_segment(vcpu
, &sregs
->fs
, VCPU_SREG_FS
);
1667 set_segment(vcpu
, &sregs
->gs
, VCPU_SREG_GS
);
1668 set_segment(vcpu
, &sregs
->ss
, VCPU_SREG_SS
);
1670 set_segment(vcpu
, &sregs
->tr
, VCPU_SREG_TR
);
1671 set_segment(vcpu
, &sregs
->ldt
, VCPU_SREG_LDTR
);
1673 dt
.limit
= sregs
->idt
.limit
;
1674 dt
.base
= sregs
->idt
.base
;
1675 kvm_arch_ops
->set_idt(vcpu
, &dt
);
1676 dt
.limit
= sregs
->gdt
.limit
;
1677 dt
.base
= sregs
->gdt
.base
;
1678 kvm_arch_ops
->set_gdt(vcpu
, &dt
);
1680 vcpu
->cr2
= sregs
->cr2
;
1681 mmu_reset_needed
|= vcpu
->cr3
!= sregs
->cr3
;
1682 vcpu
->cr3
= sregs
->cr3
;
1684 vcpu
->cr8
= sregs
->cr8
;
1686 mmu_reset_needed
|= vcpu
->shadow_efer
!= sregs
->efer
;
1687 #ifdef CONFIG_X86_64
1688 kvm_arch_ops
->set_efer(vcpu
, sregs
->efer
);
1690 vcpu
->apic_base
= sregs
->apic_base
;
1692 kvm_arch_ops
->decache_cr0_cr4_guest_bits(vcpu
);
1694 mmu_reset_needed
|= vcpu
->cr0
!= sregs
->cr0
;
1695 kvm_arch_ops
->set_cr0_no_modeswitch(vcpu
, sregs
->cr0
);
1697 mmu_reset_needed
|= vcpu
->cr4
!= sregs
->cr4
;
1698 kvm_arch_ops
->set_cr4(vcpu
, sregs
->cr4
);
1699 if (!is_long_mode(vcpu
) && is_pae(vcpu
))
1700 load_pdptrs(vcpu
, vcpu
->cr3
);
1702 if (mmu_reset_needed
)
1703 kvm_mmu_reset_context(vcpu
);
1705 memcpy(vcpu
->irq_pending
, sregs
->interrupt_bitmap
,
1706 sizeof vcpu
->irq_pending
);
1707 vcpu
->irq_summary
= 0;
1708 for (i
= 0; i
< NR_IRQ_WORDS
; ++i
)
1709 if (vcpu
->irq_pending
[i
])
1710 __set_bit(i
, &vcpu
->irq_summary
);
1718 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1719 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1721 * This list is modified at module load time to reflect the
1722 * capabilities of the host cpu.
1724 static u32 msrs_to_save
[] = {
1725 MSR_IA32_SYSENTER_CS
, MSR_IA32_SYSENTER_ESP
, MSR_IA32_SYSENTER_EIP
,
1727 #ifdef CONFIG_X86_64
1728 MSR_CSTAR
, MSR_KERNEL_GS_BASE
, MSR_SYSCALL_MASK
, MSR_LSTAR
,
1730 MSR_IA32_TIME_STAMP_COUNTER
,
1733 static unsigned num_msrs_to_save
;
1735 static u32 emulated_msrs
[] = {
1736 MSR_IA32_MISC_ENABLE
,
1739 static __init
void kvm_init_msr_list(void)
1744 for (i
= j
= 0; i
< ARRAY_SIZE(msrs_to_save
); i
++) {
1745 if (rdmsr_safe(msrs_to_save
[i
], &dummy
[0], &dummy
[1]) < 0)
1748 msrs_to_save
[j
] = msrs_to_save
[i
];
1751 num_msrs_to_save
= j
;
1755 * Adapt set_msr() to msr_io()'s calling convention
1757 static int do_set_msr(struct kvm_vcpu
*vcpu
, unsigned index
, u64
*data
)
1759 return set_msr(vcpu
, index
, *data
);
1763 * Read or write a bunch of msrs. All parameters are kernel addresses.
1765 * @return number of msrs set successfully.
1767 static int __msr_io(struct kvm
*kvm
, struct kvm_msrs
*msrs
,
1768 struct kvm_msr_entry
*entries
,
1769 int (*do_msr
)(struct kvm_vcpu
*vcpu
,
1770 unsigned index
, u64
*data
))
1772 struct kvm_vcpu
*vcpu
;
1775 if (!valid_vcpu(msrs
->vcpu
))
1778 vcpu
= vcpu_load(kvm
, msrs
->vcpu
);
1782 for (i
= 0; i
< msrs
->nmsrs
; ++i
)
1783 if (do_msr(vcpu
, entries
[i
].index
, &entries
[i
].data
))
1792 * Read or write a bunch of msrs. Parameters are user addresses.
1794 * @return number of msrs set successfully.
1796 static int msr_io(struct kvm
*kvm
, struct kvm_msrs __user
*user_msrs
,
1797 int (*do_msr
)(struct kvm_vcpu
*vcpu
,
1798 unsigned index
, u64
*data
),
1801 struct kvm_msrs msrs
;
1802 struct kvm_msr_entry
*entries
;
1807 if (copy_from_user(&msrs
, user_msrs
, sizeof msrs
))
1811 if (msrs
.nmsrs
>= MAX_IO_MSRS
)
1815 size
= sizeof(struct kvm_msr_entry
) * msrs
.nmsrs
;
1816 entries
= vmalloc(size
);
1821 if (copy_from_user(entries
, user_msrs
->entries
, size
))
1824 r
= n
= __msr_io(kvm
, &msrs
, entries
, do_msr
);
1829 if (writeback
&& copy_to_user(user_msrs
->entries
, entries
, size
))
1841 * Translate a guest virtual address to a guest physical address.
1843 static int kvm_dev_ioctl_translate(struct kvm
*kvm
, struct kvm_translation
*tr
)
1845 unsigned long vaddr
= tr
->linear_address
;
1846 struct kvm_vcpu
*vcpu
;
1849 vcpu
= vcpu_load(kvm
, tr
->vcpu
);
1852 spin_lock(&kvm
->lock
);
1853 gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, vaddr
);
1854 tr
->physical_address
= gpa
;
1855 tr
->valid
= gpa
!= UNMAPPED_GVA
;
1858 spin_unlock(&kvm
->lock
);
1864 static int kvm_dev_ioctl_interrupt(struct kvm
*kvm
, struct kvm_interrupt
*irq
)
1866 struct kvm_vcpu
*vcpu
;
1868 if (!valid_vcpu(irq
->vcpu
))
1870 if (irq
->irq
< 0 || irq
->irq
>= 256)
1872 vcpu
= vcpu_load(kvm
, irq
->vcpu
);
1876 set_bit(irq
->irq
, vcpu
->irq_pending
);
1877 set_bit(irq
->irq
/ BITS_PER_LONG
, &vcpu
->irq_summary
);
1884 static int kvm_dev_ioctl_debug_guest(struct kvm
*kvm
,
1885 struct kvm_debug_guest
*dbg
)
1887 struct kvm_vcpu
*vcpu
;
1890 if (!valid_vcpu(dbg
->vcpu
))
1892 vcpu
= vcpu_load(kvm
, dbg
->vcpu
);
1896 r
= kvm_arch_ops
->set_guest_debug(vcpu
, dbg
);
1903 static long kvm_dev_ioctl(struct file
*filp
,
1904 unsigned int ioctl
, unsigned long arg
)
1906 struct kvm
*kvm
= filp
->private_data
;
1907 void __user
*argp
= (void __user
*)arg
;
1911 case KVM_GET_API_VERSION
:
1912 r
= KVM_API_VERSION
;
1914 case KVM_CREATE_VCPU
:
1915 r
= kvm_dev_ioctl_create_vcpu(kvm
, arg
);
1920 struct kvm_run kvm_run
;
1923 if (copy_from_user(&kvm_run
, argp
, sizeof kvm_run
))
1925 r
= kvm_dev_ioctl_run(kvm
, &kvm_run
);
1926 if (r
< 0 && r
!= -EINTR
)
1928 if (copy_to_user(argp
, &kvm_run
, sizeof kvm_run
)) {
1934 case KVM_GET_REGS
: {
1935 struct kvm_regs kvm_regs
;
1938 if (copy_from_user(&kvm_regs
, argp
, sizeof kvm_regs
))
1940 r
= kvm_dev_ioctl_get_regs(kvm
, &kvm_regs
);
1944 if (copy_to_user(argp
, &kvm_regs
, sizeof kvm_regs
))
1949 case KVM_SET_REGS
: {
1950 struct kvm_regs kvm_regs
;
1953 if (copy_from_user(&kvm_regs
, argp
, sizeof kvm_regs
))
1955 r
= kvm_dev_ioctl_set_regs(kvm
, &kvm_regs
);
1961 case KVM_GET_SREGS
: {
1962 struct kvm_sregs kvm_sregs
;
1965 if (copy_from_user(&kvm_sregs
, argp
, sizeof kvm_sregs
))
1967 r
= kvm_dev_ioctl_get_sregs(kvm
, &kvm_sregs
);
1971 if (copy_to_user(argp
, &kvm_sregs
, sizeof kvm_sregs
))
1976 case KVM_SET_SREGS
: {
1977 struct kvm_sregs kvm_sregs
;
1980 if (copy_from_user(&kvm_sregs
, argp
, sizeof kvm_sregs
))
1982 r
= kvm_dev_ioctl_set_sregs(kvm
, &kvm_sregs
);
1988 case KVM_TRANSLATE
: {
1989 struct kvm_translation tr
;
1992 if (copy_from_user(&tr
, argp
, sizeof tr
))
1994 r
= kvm_dev_ioctl_translate(kvm
, &tr
);
1998 if (copy_to_user(argp
, &tr
, sizeof tr
))
2003 case KVM_INTERRUPT
: {
2004 struct kvm_interrupt irq
;
2007 if (copy_from_user(&irq
, argp
, sizeof irq
))
2009 r
= kvm_dev_ioctl_interrupt(kvm
, &irq
);
2015 case KVM_DEBUG_GUEST
: {
2016 struct kvm_debug_guest dbg
;
2019 if (copy_from_user(&dbg
, argp
, sizeof dbg
))
2021 r
= kvm_dev_ioctl_debug_guest(kvm
, &dbg
);
2027 case KVM_SET_MEMORY_REGION
: {
2028 struct kvm_memory_region kvm_mem
;
2031 if (copy_from_user(&kvm_mem
, argp
, sizeof kvm_mem
))
2033 r
= kvm_dev_ioctl_set_memory_region(kvm
, &kvm_mem
);
2038 case KVM_GET_DIRTY_LOG
: {
2039 struct kvm_dirty_log log
;
2042 if (copy_from_user(&log
, argp
, sizeof log
))
2044 r
= kvm_dev_ioctl_get_dirty_log(kvm
, &log
);
2050 r
= msr_io(kvm
, argp
, get_msr
, 1);
2053 r
= msr_io(kvm
, argp
, do_set_msr
, 0);
2055 case KVM_GET_MSR_INDEX_LIST
: {
2056 struct kvm_msr_list __user
*user_msr_list
= argp
;
2057 struct kvm_msr_list msr_list
;
2061 if (copy_from_user(&msr_list
, user_msr_list
, sizeof msr_list
))
2064 msr_list
.nmsrs
= num_msrs_to_save
+ ARRAY_SIZE(emulated_msrs
);
2065 if (copy_to_user(user_msr_list
, &msr_list
, sizeof msr_list
))
2068 if (n
< num_msrs_to_save
)
2071 if (copy_to_user(user_msr_list
->indices
, &msrs_to_save
,
2072 num_msrs_to_save
* sizeof(u32
)))
2074 if (copy_to_user(user_msr_list
->indices
2075 + num_msrs_to_save
* sizeof(u32
),
2077 ARRAY_SIZE(emulated_msrs
) * sizeof(u32
)))
2089 static struct page
*kvm_dev_nopage(struct vm_area_struct
*vma
,
2090 unsigned long address
,
2093 struct kvm
*kvm
= vma
->vm_file
->private_data
;
2094 unsigned long pgoff
;
2095 struct kvm_memory_slot
*slot
;
2098 *type
= VM_FAULT_MINOR
;
2099 pgoff
= ((address
- vma
->vm_start
) >> PAGE_SHIFT
) + vma
->vm_pgoff
;
2100 slot
= gfn_to_memslot(kvm
, pgoff
);
2102 return NOPAGE_SIGBUS
;
2103 page
= gfn_to_page(slot
, pgoff
);
2105 return NOPAGE_SIGBUS
;
2110 static struct vm_operations_struct kvm_dev_vm_ops
= {
2111 .nopage
= kvm_dev_nopage
,
2114 static int kvm_dev_mmap(struct file
*file
, struct vm_area_struct
*vma
)
2116 vma
->vm_ops
= &kvm_dev_vm_ops
;
2120 static struct file_operations kvm_chardev_ops
= {
2121 .open
= kvm_dev_open
,
2122 .release
= kvm_dev_release
,
2123 .unlocked_ioctl
= kvm_dev_ioctl
,
2124 .compat_ioctl
= kvm_dev_ioctl
,
2125 .mmap
= kvm_dev_mmap
,
2128 static struct miscdevice kvm_dev
= {
2134 static int kvm_reboot(struct notifier_block
*notifier
, unsigned long val
,
2137 if (val
== SYS_RESTART
) {
2139 * Some (well, at least mine) BIOSes hang on reboot if
2142 printk(KERN_INFO
"kvm: exiting hardware virtualization\n");
2143 on_each_cpu(kvm_arch_ops
->hardware_disable
, NULL
, 0, 1);
2148 static struct notifier_block kvm_reboot_notifier
= {
2149 .notifier_call
= kvm_reboot
,
2154 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2157 static void decache_vcpus_on_cpu(int cpu
)
2160 struct kvm_vcpu
*vcpu
;
2163 spin_lock(&kvm_lock
);
2164 list_for_each_entry(vm
, &vm_list
, vm_list
)
2165 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
2166 vcpu
= &vm
->vcpus
[i
];
2168 * If the vcpu is locked, then it is running on some
2169 * other cpu and therefore it is not cached on the
2172 * If it's not locked, check the last cpu it executed
2175 if (mutex_trylock(&vcpu
->mutex
)) {
2176 if (vcpu
->cpu
== cpu
) {
2177 kvm_arch_ops
->vcpu_decache(vcpu
);
2180 mutex_unlock(&vcpu
->mutex
);
2183 spin_unlock(&kvm_lock
);
2186 static int kvm_cpu_hotplug(struct notifier_block
*notifier
, unsigned long val
,
2192 case CPU_DOWN_PREPARE
:
2193 case CPU_UP_CANCELED
:
2194 printk(KERN_INFO
"kvm: disabling virtualization on CPU%d\n",
2196 decache_vcpus_on_cpu(cpu
);
2197 smp_call_function_single(cpu
, kvm_arch_ops
->hardware_disable
,
2201 printk(KERN_INFO
"kvm: enabling virtualization on CPU%d\n",
2203 smp_call_function_single(cpu
, kvm_arch_ops
->hardware_enable
,
2210 static struct notifier_block kvm_cpu_notifier
= {
2211 .notifier_call
= kvm_cpu_hotplug
,
2212 .priority
= 20, /* must be > scheduler priority */
2215 static __init
void kvm_init_debug(void)
2217 struct kvm_stats_debugfs_item
*p
;
2219 debugfs_dir
= debugfs_create_dir("kvm", NULL
);
2220 for (p
= debugfs_entries
; p
->name
; ++p
)
2221 p
->dentry
= debugfs_create_u32(p
->name
, 0444, debugfs_dir
,
2225 static void kvm_exit_debug(void)
2227 struct kvm_stats_debugfs_item
*p
;
2229 for (p
= debugfs_entries
; p
->name
; ++p
)
2230 debugfs_remove(p
->dentry
);
2231 debugfs_remove(debugfs_dir
);
2234 static int kvm_suspend(struct sys_device
*dev
, pm_message_t state
)
2236 decache_vcpus_on_cpu(raw_smp_processor_id());
2237 on_each_cpu(kvm_arch_ops
->hardware_disable
, NULL
, 0, 1);
2241 static int kvm_resume(struct sys_device
*dev
)
2243 on_each_cpu(kvm_arch_ops
->hardware_enable
, NULL
, 0, 1);
2247 static struct sysdev_class kvm_sysdev_class
= {
2248 set_kset_name("kvm"),
2249 .suspend
= kvm_suspend
,
2250 .resume
= kvm_resume
,
2253 static struct sys_device kvm_sysdev
= {
2255 .cls
= &kvm_sysdev_class
,
2258 hpa_t bad_page_address
;
2260 static int kvmfs_get_sb(struct file_system_type
*fs_type
, int flags
,
2261 const char *dev_name
, void *data
, struct vfsmount
*mnt
)
2263 return get_sb_pseudo(fs_type
, "kvm:", NULL
, KVMFS_MAGIC
, mnt
);
2266 static struct file_system_type kvm_fs_type
= {
2268 .get_sb
= kvmfs_get_sb
,
2269 .kill_sb
= kill_anon_super
,
2272 int kvm_init_arch(struct kvm_arch_ops
*ops
, struct module
*module
)
2277 printk(KERN_ERR
"kvm: already loaded the other module\n");
2281 if (!ops
->cpu_has_kvm_support()) {
2282 printk(KERN_ERR
"kvm: no hardware support\n");
2285 if (ops
->disabled_by_bios()) {
2286 printk(KERN_ERR
"kvm: disabled by bios\n");
2292 r
= kvm_arch_ops
->hardware_setup();
2296 on_each_cpu(kvm_arch_ops
->hardware_enable
, NULL
, 0, 1);
2297 r
= register_cpu_notifier(&kvm_cpu_notifier
);
2300 register_reboot_notifier(&kvm_reboot_notifier
);
2302 r
= sysdev_class_register(&kvm_sysdev_class
);
2306 r
= sysdev_register(&kvm_sysdev
);
2310 kvm_chardev_ops
.owner
= module
;
2312 r
= misc_register(&kvm_dev
);
2314 printk (KERN_ERR
"kvm: misc device register failed\n");
2321 sysdev_unregister(&kvm_sysdev
);
2323 sysdev_class_unregister(&kvm_sysdev_class
);
2325 unregister_reboot_notifier(&kvm_reboot_notifier
);
2326 unregister_cpu_notifier(&kvm_cpu_notifier
);
2328 on_each_cpu(kvm_arch_ops
->hardware_disable
, NULL
, 0, 1);
2329 kvm_arch_ops
->hardware_unsetup();
2333 void kvm_exit_arch(void)
2335 misc_deregister(&kvm_dev
);
2336 sysdev_unregister(&kvm_sysdev
);
2337 sysdev_class_unregister(&kvm_sysdev_class
);
2338 unregister_reboot_notifier(&kvm_reboot_notifier
);
2339 unregister_cpu_notifier(&kvm_cpu_notifier
);
2340 on_each_cpu(kvm_arch_ops
->hardware_disable
, NULL
, 0, 1);
2341 kvm_arch_ops
->hardware_unsetup();
2342 kvm_arch_ops
= NULL
;
2345 static __init
int kvm_init(void)
2347 static struct page
*bad_page
;
2350 r
= register_filesystem(&kvm_fs_type
);
2354 kvmfs_mnt
= kern_mount(&kvm_fs_type
);
2355 r
= PTR_ERR(kvmfs_mnt
);
2356 if (IS_ERR(kvmfs_mnt
))
2360 kvm_init_msr_list();
2362 if ((bad_page
= alloc_page(GFP_KERNEL
)) == NULL
) {
2367 bad_page_address
= page_to_pfn(bad_page
) << PAGE_SHIFT
;
2368 memset(__va(bad_page_address
), 0, PAGE_SIZE
);
2376 unregister_filesystem(&kvm_fs_type
);
2381 static __exit
void kvm_exit(void)
2384 __free_page(pfn_to_page(bad_page_address
>> PAGE_SHIFT
));
2386 unregister_filesystem(&kvm_fs_type
);
2389 module_init(kvm_init
)
2390 module_exit(kvm_exit
)
2392 EXPORT_SYMBOL_GPL(kvm_init_arch
);
2393 EXPORT_SYMBOL_GPL(kvm_exit_arch
);