drivers/kvm/paging_tmpl.h

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * MMU support
   8  *
   9  * Copyright (C) 2006 Qumranet, Inc.
  10  *
  11  * Authors:
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *   Avi Kivity   <avi@qumranet.com>
  14  *
  15  * This work is licensed under the terms of the GNU GPL, version 2.  See
  16  * the COPYING file in the top-level directory.
  17  *
  18  */
  19
  20 /*
  21  * We need the mmu code to access both 32-bit and 64-bit guest ptes,
  22  * so the code in this file is compiled twice, once per pte size.
  23  */
  24
  25 #if PTTYPE == 64
  26         #define pt_element_t u64
  27         #define guest_walker guest_walker64
  28         #define FNAME(name) paging##64_##name
  29         #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
  30         #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK
  31         #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
  32         #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
  33         #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level)
  34         #define PT_LEVEL_BITS PT64_LEVEL_BITS
  35         #ifdef CONFIG_X86_64
  36         #define PT_MAX_FULL_LEVELS 4
  37         #define CMPXCHG cmpxchg
  38         #else
  39         #define CMPXCHG cmpxchg64
  40         #define PT_MAX_FULL_LEVELS 2
  41         #endif
  42 #elif PTTYPE == 32
  43         #define pt_element_t u32
  44         #define guest_walker guest_walker32
  45         #define FNAME(name) paging##32_##name
  46         #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
  47         #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK
  48         #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
  49         #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
  50         #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level)
  51         #define PT_LEVEL_BITS PT32_LEVEL_BITS
  52         #define PT_MAX_FULL_LEVELS 2
  53         #define CMPXCHG cmpxchg
  54 #else
  55         #error Invalid PTTYPE value
  56 #endif
  57
  58 #define gpte_to_gfn FNAME(gpte_to_gfn)
  59 #define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde)
  60
  61 /*
  62  * The guest_walker structure emulates the behavior of the hardware page
  63  * table walker.
  64  */
  65 struct guest_walker {
  66         int level;
  67         gfn_t table_gfn[PT_MAX_FULL_LEVELS];
  68         pt_element_t pte;
  69         unsigned pt_access;
  70         unsigned pte_access;
  71         gfn_t gfn;
  72         u32 error_code;
  73 };
  74
  75 static gfn_t gpte_to_gfn(pt_element_t gpte)
  76 {
  77         return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT;
  78 }
  79
  80 static gfn_t gpte_to_gfn_pde(pt_element_t gpte)
  81 {
  82         return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT;
  83 }
  84
  85 static bool FNAME(cmpxchg_gpte)(struct kvm *kvm,
  86                          gfn_t table_gfn, unsigned index,
  87                          pt_element_t orig_pte, pt_element_t new_pte)
  88 {
  89         pt_element_t ret;
  90         pt_element_t *table;
  91         struct page *page;
  92
  93         page = gfn_to_page(kvm, table_gfn);
  94         table = kmap_atomic(page, KM_USER0);
  95
  96         ret = CMPXCHG(&table[index], orig_pte, new_pte);
  97
  98         kunmap_atomic(table, KM_USER0);
  99
 100         kvm_release_page_dirty(page);
 101
 102         return (ret != orig_pte);
 103 }
 104
 105 static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte)
 106 {
 107         unsigned access;
 108
 109         access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
 110 #if PTTYPE == 64
 111         if (is_nx(vcpu))
 112                 access &= ~(gpte >> PT64_NX_SHIFT);
 113 #endif
 114         return access;
 115 }
 116
 117 /*
 118  * Fetch a guest pte for a guest virtual address
 119  */
 120 static int FNAME(walk_addr)(struct guest_walker *walker,
 121                             struct kvm_vcpu *vcpu, gva_t addr,
 122                             int write_fault, int user_fault, int fetch_fault)
 123 {
 124         pt_element_t pte;
 125         gfn_t table_gfn;
 126         unsigned index, pt_access, pte_access;
 127         gpa_t pte_gpa;
 128
 129         pgprintk("%s: addr %lx\n", __FUNCTION__, addr);
 130 walk:
 131         walker->level = vcpu->mmu.root_level;
 132         pte = vcpu->cr3;
 133 #if PTTYPE == 64
 134         if (!is_long_mode(vcpu)) {
 135                 pte = vcpu->pdptrs[(addr >> 30) & 3];
 136                 if (!is_present_pte(pte))
 137                         goto not_present;
 138                 --walker->level;
 139         }
 140 #endif
 141         ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
 142                (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0);
 143
 144         pt_access = ACC_ALL;
 145
 146         for (;;) {
 147                 index = PT_INDEX(addr, walker->level);
 148
 149                 table_gfn = gpte_to_gfn(pte);
 150                 pte_gpa = gfn_to_gpa(table_gfn);
 151                 pte_gpa += index * sizeof(pt_element_t);
 152                 walker->table_gfn[walker->level - 1] = table_gfn;
 153                 pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__,
 154                          walker->level - 1, table_gfn);
 155
 156                 kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte));
 157
 158                 if (!is_present_pte(pte))
 159                         goto not_present;
 160
 161                 if (write_fault && !is_writeble_pte(pte))
 162                         if (user_fault || is_write_protection(vcpu))
 163                                 goto access_error;
 164
 165                 if (user_fault && !(pte & PT_USER_MASK))
 166                         goto access_error;
 167
 168 #if PTTYPE == 64
 169                 if (fetch_fault && is_nx(vcpu) && (pte & PT64_NX_MASK))
 170                         goto access_error;
 171 #endif
 172
 173                 if (!(pte & PT_ACCESSED_MASK)) {
 174                         mark_page_dirty(vcpu->kvm, table_gfn);
 175                         if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn,
 176                             index, pte, pte|PT_ACCESSED_MASK))
 177                                 goto walk;
 178                         pte |= PT_ACCESSED_MASK;
 179                 }
 180
 181                 pte_access = pt_access & FNAME(gpte_access)(vcpu, pte);
 182
 183                 if (walker->level == PT_PAGE_TABLE_LEVEL) {
 184                         walker->gfn = gpte_to_gfn(pte);
 185                         break;
 186                 }
 187
 188                 if (walker->level == PT_DIRECTORY_LEVEL
 189                     && (pte & PT_PAGE_SIZE_MASK)
 190                     && (PTTYPE == 64 || is_pse(vcpu))) {
 191                         walker->gfn = gpte_to_gfn_pde(pte);
 192                         walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL);
 193                         if (PTTYPE == 32 && is_cpuid_PSE36())
 194                                 walker->gfn += pse36_gfn_delta(pte);
 195                         break;
 196                 }
 197
 198                 pt_access = pte_access;
 199                 --walker->level;
 200         }
 201
 202         if (write_fault && !is_dirty_pte(pte)) {
 203                 bool ret;
 204
 205                 mark_page_dirty(vcpu->kvm, table_gfn);
 206                 ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte,
 207                             pte|PT_DIRTY_MASK);
 208                 if (ret)
 209                         goto walk;
 210                 pte |= PT_DIRTY_MASK;
 211                 kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte));
 212         }
 213
 214         walker->pte = pte;
 215         walker->pt_access = pt_access;
 216         walker->pte_access = pte_access;
 217         pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
 218                  __FUNCTION__, (u64)pte, pt_access, pte_access);
 219         return 1;
 220
 221 not_present:
 222         walker->error_code = 0;
 223         goto err;
 224
 225 access_error:
 226         walker->error_code = PFERR_PRESENT_MASK;
 227
 228 err:
 229         if (write_fault)
 230                 walker->error_code |= PFERR_WRITE_MASK;
 231         if (user_fault)
 232                 walker->error_code |= PFERR_USER_MASK;
 233         if (fetch_fault)
 234                 walker->error_code |= PFERR_FETCH_MASK;
 235         return 0;
 236 }
 237
 238 static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte,
 239                            u64 *shadow_pte, unsigned pt_access,
 240                            unsigned pte_access,
 241                            int user_fault, int write_fault,
 242                            int *ptwrite, struct guest_walker *walker,
 243                            gfn_t gfn)
 244 {
 245         int dirty = gpte & PT_DIRTY_MASK;
 246         u64 spte;
 247         int was_rmapped = is_rmap_pte(*shadow_pte);
 248         struct page *page;
 249
 250         pgprintk("%s: spte %llx gpte %llx access %x write_fault %d"
 251                  " user_fault %d gfn %lx\n",
 252                  __FUNCTION__, *shadow_pte, (u64)gpte, pt_access,
 253                  write_fault, user_fault, gfn);
 254
 255         /*
 256          * We don't set the accessed bit, since we sometimes want to see
 257          * whether the guest actually used the pte (in order to detect
 258          * demand paging).
 259          */
 260         spte = PT_PRESENT_MASK | PT_DIRTY_MASK;
 261         spte |= gpte & PT64_NX_MASK;
 262         if (!dirty)
 263                 pte_access &= ~ACC_WRITE_MASK;
 264         if (!(pte_access & ACC_EXEC_MASK))
 265                 spte |= PT64_NX_MASK;
 266
 267         page = gfn_to_page(vcpu->kvm, gfn);
 268
 269         spte |= PT_PRESENT_MASK;
 270         if (pte_access & ACC_USER_MASK)
 271                 spte |= PT_USER_MASK;
 272
 273         if (is_error_page(page)) {
 274                 set_shadow_pte(shadow_pte,
 275                                shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK);
 276                 kvm_release_page_clean(page);
 277                 return;
 278         }
 279
 280         spte |= page_to_phys(page);
 281
 282         if ((pte_access & ACC_WRITE_MASK)
 283             || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
 284                 struct kvm_mmu_page *shadow;
 285
 286                 spte |= PT_WRITABLE_MASK;
 287                 if (user_fault) {
 288                         mmu_unshadow(vcpu->kvm, gfn);
 289                         goto unshadowed;
 290                 }
 291
 292                 shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
 293                 if (shadow) {
 294                         pgprintk("%s: found shadow page for %lx, marking ro\n",
 295                                  __FUNCTION__, gfn);
 296                         pte_access &= ~ACC_WRITE_MASK;
 297                         if (is_writeble_pte(spte)) {
 298                                 spte &= ~PT_WRITABLE_MASK;
 299                                 kvm_x86_ops->tlb_flush(vcpu);
 300                         }
 301                         if (write_fault)
 302                                 *ptwrite = 1;
 303                 }
 304         }
 305
 306 unshadowed:
 307
 308         if (pte_access & ACC_WRITE_MASK)
 309                 mark_page_dirty(vcpu->kvm, gfn);
 310
 311         pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte);
 312         set_shadow_pte(shadow_pte, spte);
 313         page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
 314         if (!was_rmapped) {
 315                 rmap_add(vcpu, shadow_pte, gfn);
 316                 if (!is_rmap_pte(*shadow_pte))
 317                         kvm_release_page_clean(page);
 318         }
 319         else
 320                 kvm_release_page_clean(page);
 321         if (!ptwrite || !*ptwrite)
 322                 vcpu->last_pte_updated = shadow_pte;
 323 }
 324
 325 static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page,
 326                               u64 *spte, const void *pte, int bytes,
 327                               int offset_in_pte)
 328 {
 329         pt_element_t gpte;
 330
 331         gpte = *(const pt_element_t *)pte;
 332         if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) {
 333                 if (!offset_in_pte && !is_present_pte(gpte))
 334                         set_shadow_pte(spte, shadow_notrap_nonpresent_pte);
 335                 return;
 336         }
 337         if (bytes < sizeof(pt_element_t))
 338                 return;
 339         pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte);
 340         FNAME(set_pte)(vcpu, gpte, spte, ACC_ALL, ACC_ALL,
 341                        0, 0, NULL, NULL, gpte_to_gfn(gpte));
 342 }
 343
 344 /*
 345  * Fetch a shadow pte for a specific level in the paging hierarchy.
 346  */
 347 static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
 348                          struct guest_walker *walker,
 349                          int user_fault, int write_fault, int *ptwrite)
 350 {
 351         hpa_t shadow_addr;
 352         int level;
 353         u64 *shadow_ent;
 354         unsigned access = walker->pt_access;
 355
 356         if (!is_present_pte(walker->pte))
 357                 return NULL;
 358
 359         shadow_addr = vcpu->mmu.root_hpa;
 360         level = vcpu->mmu.shadow_root_level;
 361         if (level == PT32E_ROOT_LEVEL) {
 362                 shadow_addr = vcpu->mmu.pae_root[(addr >> 30) & 3];
 363                 shadow_addr &= PT64_BASE_ADDR_MASK;
 364                 --level;
 365         }
 366
 367         for (; ; level--) {
 368                 u32 index = SHADOW_PT_INDEX(addr, level);
 369                 struct kvm_mmu_page *shadow_page;
 370                 u64 shadow_pte;
 371                 int metaphysical;
 372                 gfn_t table_gfn;
 373
 374                 shadow_ent = ((u64 *)__va(shadow_addr)) + index;
 375                 if (is_shadow_present_pte(*shadow_ent)) {
 376                         if (level == PT_PAGE_TABLE_LEVEL)
 377                                 break;
 378                         shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK;
 379                         continue;
 380                 }
 381
 382                 if (level == PT_PAGE_TABLE_LEVEL)
 383                         break;
 384
 385                 if (level - 1 == PT_PAGE_TABLE_LEVEL
 386                     && walker->level == PT_DIRECTORY_LEVEL) {
 387                         metaphysical = 1;
 388                         if (!is_dirty_pte(walker->pte))
 389                                 access &= ~ACC_WRITE_MASK;
 390                         table_gfn = gpte_to_gfn(walker->pte);
 391                 } else {
 392                         metaphysical = 0;
 393                         table_gfn = walker->table_gfn[level - 2];
 394                 }
 395                 shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1,
 396                                                metaphysical, access,
 397                                                shadow_ent);
 398                 shadow_addr = __pa(shadow_page->spt);
 399                 shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK
 400                         | PT_WRITABLE_MASK | PT_USER_MASK;
 401                 *shadow_ent = shadow_pte;
 402         }
 403
 404         FNAME(set_pte)(vcpu, walker->pte, shadow_ent,
 405                        access, walker->pte_access & access,
 406                        user_fault, write_fault,
 407                        ptwrite, walker, walker->gfn);
 408
 409         return shadow_ent;
 410 }
 411
 412 /*
 413  * Page fault handler.  There are several causes for a page fault:
 414  *   - there is no shadow pte for the guest pte
 415  *   - write access through a shadow pte marked read only so that we can set
 416  *     the dirty bit
 417  *   - write access to a shadow pte marked read only so we can update the page
 418  *     dirty bitmap, when userspace requests it
 419  *   - mmio access; in this case we will never install a present shadow pte
 420  *   - normal guest page fault due to the guest pte marked not present, not
 421  *     writable, or not executable
 422  *
 423  *  Returns: 1 if we need to emulate the instruction, 0 otherwise, or
 424  *           a negative value on error.
 425  */
 426 static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 427                                u32 error_code)
 428 {
 429         int write_fault = error_code & PFERR_WRITE_MASK;
 430         int user_fault = error_code & PFERR_USER_MASK;
 431         int fetch_fault = error_code & PFERR_FETCH_MASK;
 432         struct guest_walker walker;
 433         u64 *shadow_pte;
 434         int write_pt = 0;
 435         int r;
 436
 437         pgprintk("%s: addr %lx err %x\n", __FUNCTION__, addr, error_code);
 438         kvm_mmu_audit(vcpu, "pre page fault");
 439
 440         r = mmu_topup_memory_caches(vcpu);
 441         if (r)
 442                 return r;
 443
 444         /*
 445          * Look up the shadow pte for the faulting address.
 446          */
 447         r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault,
 448                              fetch_fault);
 449
 450         /*
 451          * The page is not mapped by the guest.  Let the guest handle it.
 452          */
 453         if (!r) {
 454                 pgprintk("%s: guest page fault\n", __FUNCTION__);
 455                 inject_page_fault(vcpu, addr, walker.error_code);
 456                 vcpu->last_pt_write_count = 0; /* reset fork detector */
 457                 return 0;
 458         }
 459
 460         shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
 461                                   &write_pt);
 462         pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__,
 463                  shadow_pte, *shadow_pte, write_pt);
 464
 465         if (!write_pt)
 466                 vcpu->last_pt_write_count = 0; /* reset fork detector */
 467
 468         /*
 469          * mmio: emulate if accessible, otherwise its a guest fault.
 470          */
 471         if (is_io_pte(*shadow_pte))
 472                 return 1;
 473
 474         ++vcpu->stat.pf_fixed;
 475         kvm_mmu_audit(vcpu, "post page fault (fixed)");
 476
 477         return write_pt;
 478 }
 479
 480 static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr)
 481 {
 482         struct guest_walker walker;
 483         gpa_t gpa = UNMAPPED_GVA;
 484         int r;
 485
 486         r = FNAME(walk_addr)(&walker, vcpu, vaddr, 0, 0, 0);
 487
 488         if (r) {
 489                 gpa = gfn_to_gpa(walker.gfn);
 490                 gpa |= vaddr & ~PAGE_MASK;
 491         }
 492
 493         return gpa;
 494 }
 495
 496 static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu,
 497                                  struct kvm_mmu_page *sp)
 498 {
 499         int i, offset = 0;
 500         pt_element_t *gpt;
 501         struct page *page;
 502
 503         if (sp->role.metaphysical
 504             || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) {
 505                 nonpaging_prefetch_page(vcpu, sp);
 506                 return;
 507         }
 508
 509         if (PTTYPE == 32)
 510                 offset = sp->role.quadrant << PT64_LEVEL_BITS;
 511         page = gfn_to_page(vcpu->kvm, sp->gfn);
 512         gpt = kmap_atomic(page, KM_USER0);
 513         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
 514                 if (is_present_pte(gpt[offset + i]))
 515                         sp->spt[i] = shadow_trap_nonpresent_pte;
 516                 else
 517                         sp->spt[i] = shadow_notrap_nonpresent_pte;
 518         kunmap_atomic(gpt, KM_USER0);
 519         kvm_release_page_clean(page);
 520 }
 521
 522 #undef pt_element_t
 523 #undef guest_walker
 524 #undef FNAME
 525 #undef PT_BASE_ADDR_MASK
 526 #undef PT_INDEX
 527 #undef SHADOW_PT_INDEX
 528 #undef PT_LEVEL_MASK
 529 #undef PT_DIR_BASE_ADDR_MASK
 530 #undef PT_LEVEL_BITS
 531 #undef PT_MAX_FULL_LEVELS
 532 #undef gpte_to_gfn
 533 #undef gpte_to_gfn_pde
 534 #undef CMPXCHG