arch/mips/kvm/tlb.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
   7  * TLB handlers run from KSEG0
   8  *
   9  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
  10  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  11  */
  12
  13 #include <linux/sched.h>
  14 #include <linux/smp.h>
  15 #include <linux/mm.h>
  16 #include <linux/delay.h>
  17 #include <linux/export.h>
  18 #include <linux/kvm_host.h>
  19 #include <linux/srcu.h>
  20
  21 #include <asm/cpu.h>
  22 #include <asm/bootinfo.h>
  23 #include <asm/mmu_context.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/cacheflush.h>
  26 #include <asm/tlb.h>
  27 #include <asm/tlbdebug.h>
  28
  29 #undef CONFIG_MIPS_MT
  30 #include <asm/r4kcache.h>
  31 #define CONFIG_MIPS_MT
  32
  33 #define KVM_GUEST_PC_TLB    0
  34 #define KVM_GUEST_SP_TLB    1
  35
  36 atomic_t kvm_mips_instance;
  37 EXPORT_SYMBOL_GPL(kvm_mips_instance);
  38
  39 static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
  40 {
  41         int cpu = smp_processor_id();
  42
  43         return vcpu->arch.guest_kernel_asid[cpu] &
  44                         cpu_asid_mask(&cpu_data[cpu]);
  45 }
  46
  47 static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
  48 {
  49         int cpu = smp_processor_id();
  50
  51         return vcpu->arch.guest_user_asid[cpu] &
  52                         cpu_asid_mask(&cpu_data[cpu]);
  53 }
  54
  55 inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu)
  56 {
  57         return vcpu->kvm->arch.commpage_tlb;
  58 }
  59
  60 /* Structure defining an tlb entry data set. */
  61
  62 void kvm_mips_dump_host_tlbs(void)
  63 {
  64         unsigned long flags;
  65
  66         local_irq_save(flags);
  67
  68         kvm_info("HOST TLBs:\n");
  69         dump_tlb_regs();
  70         pr_info("\n");
  71         dump_tlb_all();
  72
  73         local_irq_restore(flags);
  74 }
  75 EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);
  76
  77 void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
  78 {
  79         struct mips_coproc *cop0 = vcpu->arch.cop0;
  80         struct kvm_mips_tlb tlb;
  81         int i;
  82
  83         kvm_info("Guest TLBs:\n");
  84         kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
  85
  86         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
  87                 tlb = vcpu->arch.guest_tlb[i];
  88                 kvm_info("TLB%c%3d Hi 0x%08lx ",
  89                          (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & MIPS3_PG_V
  90                                                         ? ' ' : '*',
  91                          i, tlb.tlb_hi);
  92                 kvm_info("Lo0=0x%09llx %c%c attr %lx ",
  93                          (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]),
  94                          (tlb.tlb_lo[0] & MIPS3_PG_D) ? 'D' : ' ',
  95                          (tlb.tlb_lo[0] & MIPS3_PG_G) ? 'G' : ' ',
  96                          (tlb.tlb_lo[0] >> 3) & 7);
  97                 kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
  98                          (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]),
  99                          (tlb.tlb_lo[1] & MIPS3_PG_D) ? 'D' : ' ',
 100                          (tlb.tlb_lo[1] & MIPS3_PG_G) ? 'G' : ' ',
 101                          (tlb.tlb_lo[1] >> 3) & 7, tlb.tlb_mask);
 102         }
 103 }
 104 EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);
 105
 106 /* XXXKYMA: Must be called with interrupts disabled */
 107 /* set flush_dcache_mask == 0 if no dcache flush required */
 108 int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi,
 109                             unsigned long entrylo0, unsigned long entrylo1,
 110                             int flush_dcache_mask)
 111 {
 112         unsigned long flags;
 113         unsigned long old_entryhi;
 114         int idx;
 115
 116         local_irq_save(flags);
 117
 118         old_entryhi = read_c0_entryhi();
 119         write_c0_entryhi(entryhi);
 120         mtc0_tlbw_hazard();
 121
 122         tlb_probe();
 123         tlb_probe_hazard();
 124         idx = read_c0_index();
 125
 126         if (idx > current_cpu_data.tlbsize) {
 127                 kvm_err("%s: Invalid Index: %d\n", __func__, idx);
 128                 kvm_mips_dump_host_tlbs();
 129                 local_irq_restore(flags);
 130                 return -1;
 131         }
 132
 133         write_c0_entrylo0(entrylo0);
 134         write_c0_entrylo1(entrylo1);
 135         mtc0_tlbw_hazard();
 136
 137         if (idx < 0)
 138                 tlb_write_random();
 139         else
 140                 tlb_write_indexed();
 141         tlbw_use_hazard();
 142
 143         kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n",
 144                   vcpu->arch.pc, idx, read_c0_entryhi(),
 145                   read_c0_entrylo0(), read_c0_entrylo1());
 146
 147         /* Flush D-cache */
 148         if (flush_dcache_mask) {
 149                 if (entrylo0 & MIPS3_PG_V) {
 150                         ++vcpu->stat.flush_dcache_exits;
 151                         flush_data_cache_page((entryhi & VPN2_MASK) &
 152                                               ~flush_dcache_mask);
 153                 }
 154                 if (entrylo1 & MIPS3_PG_V) {
 155                         ++vcpu->stat.flush_dcache_exits;
 156                         flush_data_cache_page(((entryhi & VPN2_MASK) &
 157                                                ~flush_dcache_mask) |
 158                                               (0x1 << PAGE_SHIFT));
 159                 }
 160         }
 161
 162         /* Restore old ASID */
 163         write_c0_entryhi(old_entryhi);
 164         mtc0_tlbw_hazard();
 165         local_irq_restore(flags);
 166         return 0;
 167 }
 168 EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write);
 169
 170 int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
 171         struct kvm_vcpu *vcpu)
 172 {
 173         kvm_pfn_t pfn0, pfn1;
 174         unsigned long flags, old_entryhi = 0, vaddr = 0;
 175         unsigned long entrylo0 = 0, entrylo1 = 0;
 176
 177         pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT;
 178         pfn1 = 0;
 179         entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) |
 180                    (1 << 2) | (0x1 << 1);
 181         entrylo1 = 0;
 182
 183         local_irq_save(flags);
 184
 185         old_entryhi = read_c0_entryhi();
 186         vaddr = badvaddr & (PAGE_MASK << 1);
 187         write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu));
 188         write_c0_entrylo0(entrylo0);
 189         write_c0_entrylo1(entrylo1);
 190         write_c0_index(kvm_mips_get_commpage_asid(vcpu));
 191         mtc0_tlbw_hazard();
 192         tlb_write_indexed();
 193         tlbw_use_hazard();
 194
 195         kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n",
 196                   vcpu->arch.pc, read_c0_index(), read_c0_entryhi(),
 197                   read_c0_entrylo0(), read_c0_entrylo1());
 198
 199         /* Restore old ASID */
 200         write_c0_entryhi(old_entryhi);
 201         mtc0_tlbw_hazard();
 202         local_irq_restore(flags);
 203
 204         return 0;
 205 }
 206 EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault);
 207
 208 int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
 209 {
 210         int i;
 211         int index = -1;
 212         struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
 213
 214         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
 215                 if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
 216                     TLB_HI_ASID_HIT(tlb[i], entryhi)) {
 217                         index = i;
 218                         break;
 219                 }
 220         }
 221
 222         kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
 223                   __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]);
 224
 225         return index;
 226 }
 227 EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);
 228
 229 int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr)
 230 {
 231         unsigned long old_entryhi, flags;
 232         int idx;
 233
 234         local_irq_save(flags);
 235
 236         old_entryhi = read_c0_entryhi();
 237
 238         if (KVM_GUEST_KERNEL_MODE(vcpu))
 239                 write_c0_entryhi((vaddr & VPN2_MASK) |
 240                                  kvm_mips_get_kernel_asid(vcpu));
 241         else {
 242                 write_c0_entryhi((vaddr & VPN2_MASK) |
 243                                  kvm_mips_get_user_asid(vcpu));
 244         }
 245
 246         mtc0_tlbw_hazard();
 247
 248         tlb_probe();
 249         tlb_probe_hazard();
 250         idx = read_c0_index();
 251
 252         /* Restore old ASID */
 253         write_c0_entryhi(old_entryhi);
 254         mtc0_tlbw_hazard();
 255
 256         local_irq_restore(flags);
 257
 258         kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx);
 259
 260         return idx;
 261 }
 262 EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup);
 263
 264 int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
 265 {
 266         int idx;
 267         unsigned long flags, old_entryhi;
 268
 269         local_irq_save(flags);
 270
 271         old_entryhi = read_c0_entryhi();
 272
 273         write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu));
 274         mtc0_tlbw_hazard();
 275
 276         tlb_probe();
 277         tlb_probe_hazard();
 278         idx = read_c0_index();
 279
 280         if (idx >= current_cpu_data.tlbsize)
 281                 BUG();
 282
 283         if (idx > 0) {
 284                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 285                 write_c0_entrylo0(0);
 286                 write_c0_entrylo1(0);
 287                 mtc0_tlbw_hazard();
 288
 289                 tlb_write_indexed();
 290                 tlbw_use_hazard();
 291         }
 292
 293         write_c0_entryhi(old_entryhi);
 294         mtc0_tlbw_hazard();
 295
 296         local_irq_restore(flags);
 297
 298         if (idx > 0)
 299                 kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__,
 300                           (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx);
 301
 302         return 0;
 303 }
 304 EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);
 305
 306 void kvm_mips_flush_host_tlb(int skip_kseg0)
 307 {
 308         unsigned long flags;
 309         unsigned long old_entryhi, entryhi;
 310         unsigned long old_pagemask;
 311         int entry = 0;
 312         int maxentry = current_cpu_data.tlbsize;
 313
 314         local_irq_save(flags);
 315
 316         old_entryhi = read_c0_entryhi();
 317         old_pagemask = read_c0_pagemask();
 318
 319         /* Blast 'em all away. */
 320         for (entry = 0; entry < maxentry; entry++) {
 321                 write_c0_index(entry);
 322
 323                 if (skip_kseg0) {
 324                         mtc0_tlbr_hazard();
 325                         tlb_read();
 326                         tlb_read_hazard();
 327
 328                         entryhi = read_c0_entryhi();
 329
 330                         /* Don't blow away guest kernel entries */
 331                         if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0)
 332                                 continue;
 333                 }
 334
 335                 /* Make sure all entries differ. */
 336                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 337                 write_c0_entrylo0(0);
 338                 write_c0_entrylo1(0);
 339                 mtc0_tlbw_hazard();
 340
 341                 tlb_write_indexed();
 342                 tlbw_use_hazard();
 343         }
 344
 345         write_c0_entryhi(old_entryhi);
 346         write_c0_pagemask(old_pagemask);
 347         mtc0_tlbw_hazard();
 348
 349         local_irq_restore(flags);
 350 }
 351 EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb);
 352
 353 void kvm_local_flush_tlb_all(void)
 354 {
 355         unsigned long flags;
 356         unsigned long old_ctx;
 357         int entry = 0;
 358
 359         local_irq_save(flags);
 360         /* Save old context and create impossible VPN2 value */
 361         old_ctx = read_c0_entryhi();
 362         write_c0_entrylo0(0);
 363         write_c0_entrylo1(0);
 364
 365         /* Blast 'em all away. */
 366         while (entry < current_cpu_data.tlbsize) {
 367                 /* Make sure all entries differ. */
 368                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 369                 write_c0_index(entry);
 370                 mtc0_tlbw_hazard();
 371                 tlb_write_indexed();
 372                 tlbw_use_hazard();
 373                 entry++;
 374         }
 375         write_c0_entryhi(old_ctx);
 376         mtc0_tlbw_hazard();
 377
 378         local_irq_restore(flags);
 379 }
 380 EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all);