arch/mips/mm/tlb-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   7  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
   8  * Carsten Langgaard, carstenl@mips.com
   9  * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/sched.h>
  13 #include <linux/smp.h>
  14 #include <linux/mm.h>
  15 #include <linux/hugetlb.h>
  16
  17 #include <asm/cpu.h>
  18 #include <asm/bootinfo.h>
  19 #include <asm/mmu_context.h>
  20 #include <asm/pgtable.h>
  21 #include <asm/tlbmisc.h>
  22
  23 extern void build_tlb_refill_handler(void);
  24
  25 /*
  26  * Make sure all entries differ.  If they're not different
  27  * MIPS32 will take revenge ...
  28  */
  29 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
  30
  31 /* Atomicity and interruptability */
  32 #ifdef CONFIG_MIPS_MT_SMTC
  33
  34 #include <asm/smtc.h>
  35 #include <asm/mipsmtregs.h>
  36
  37 #define ENTER_CRITICAL(flags) \
  38         { \
  39         unsigned int mvpflags; \
  40         local_irq_save(flags);\
  41         mvpflags = dvpe()
  42 #define EXIT_CRITICAL(flags) \
  43         evpe(mvpflags); \
  44         local_irq_restore(flags); \
  45         }
  46 #else
  47
  48 #define ENTER_CRITICAL(flags) local_irq_save(flags)
  49 #define EXIT_CRITICAL(flags) local_irq_restore(flags)
  50
  51 #endif /* CONFIG_MIPS_MT_SMTC */
  52
  53 #if defined(CONFIG_CPU_LOONGSON2)
  54 /*
  55  * LOONGSON2 has a 4 entry itlb which is a subset of dtlb,
  56  * unfortrunately, itlb is not totally transparent to software.
  57  */
  58 #define FLUSH_ITLB write_c0_diag(4);
  59
  60 #define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC)  write_c0_diag(4); }
  61
  62 #else
  63
  64 #define FLUSH_ITLB
  65 #define FLUSH_ITLB_VM(vma)
  66
  67 #endif
  68
  69 void local_flush_tlb_all(void)
  70 {
  71         unsigned long flags;
  72         unsigned long old_ctx;
  73         int entry;
  74
  75         ENTER_CRITICAL(flags);
  76         /* Save old context and create impossible VPN2 value */
  77         old_ctx = read_c0_entryhi();
  78         write_c0_entrylo0(0);
  79         write_c0_entrylo1(0);
  80
  81         entry = read_c0_wired();
  82
  83         /* Blast 'em all away. */
  84         while (entry < current_cpu_data.tlbsize) {
  85                 /* Make sure all entries differ. */
  86                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
  87                 write_c0_index(entry);
  88                 mtc0_tlbw_hazard();
  89                 tlb_write_indexed();
  90                 entry++;
  91         }
  92         tlbw_use_hazard();
  93         write_c0_entryhi(old_ctx);
  94         FLUSH_ITLB;
  95         EXIT_CRITICAL(flags);
  96 }
  97
  98 /* All entries common to a mm share an asid.  To effectively flush
  99    these entries, we just bump the asid. */
 100 void local_flush_tlb_mm(struct mm_struct *mm)
 101 {
 102         int cpu;
 103
 104         preempt_disable();
 105
 106         cpu = smp_processor_id();
 107
 108         if (cpu_context(cpu, mm) != 0) {
 109                 drop_mmu_context(mm, cpu);
 110         }
 111
 112         preempt_enable();
 113 }
 114
 115 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 116         unsigned long end)
 117 {
 118         struct mm_struct *mm = vma->vm_mm;
 119         int cpu = smp_processor_id();
 120
 121         if (cpu_context(cpu, mm) != 0) {
 122                 unsigned long size, flags;
 123                 int huge = is_vm_hugetlb_page(vma);
 124
 125                 ENTER_CRITICAL(flags);
 126                 if (huge) {
 127                         start = round_down(start, HPAGE_SIZE);
 128                         end = round_up(end, HPAGE_SIZE);
 129                         size = (end - start) >> HPAGE_SHIFT;
 130                 } else {
 131                         start = round_down(start, PAGE_SIZE << 1);
 132                         end = round_up(end, PAGE_SIZE << 1);
 133                         size = (end - start) >> (PAGE_SHIFT + 1);
 134                 }
 135                 if (size <= current_cpu_data.tlbsize/2) {
 136                         int oldpid = read_c0_entryhi();
 137                         int newpid = cpu_asid(cpu, mm);
 138
 139                         while (start < end) {
 140                                 int idx;
 141
 142                                 write_c0_entryhi(start | newpid);
 143                                 if (huge)
 144                                         start += HPAGE_SIZE;
 145                                 else
 146                                         start += (PAGE_SIZE << 1);
 147                                 mtc0_tlbw_hazard();
 148                                 tlb_probe();
 149                                 tlb_probe_hazard();
 150                                 idx = read_c0_index();
 151                                 write_c0_entrylo0(0);
 152                                 write_c0_entrylo1(0);
 153                                 if (idx < 0)
 154                                         continue;
 155                                 /* Make sure all entries differ. */
 156                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 157                                 mtc0_tlbw_hazard();
 158                                 tlb_write_indexed();
 159                         }
 160                         tlbw_use_hazard();
 161                         write_c0_entryhi(oldpid);
 162                 } else {
 163                         drop_mmu_context(mm, cpu);
 164                 }
 165                 FLUSH_ITLB;
 166                 EXIT_CRITICAL(flags);
 167         }
 168 }
 169
 170 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 171 {
 172         unsigned long size, flags;
 173
 174         ENTER_CRITICAL(flags);
 175         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 176         size = (size + 1) >> 1;
 177         if (size <= current_cpu_data.tlbsize / 2) {
 178                 int pid = read_c0_entryhi();
 179
 180                 start &= (PAGE_MASK << 1);
 181                 end += ((PAGE_SIZE << 1) - 1);
 182                 end &= (PAGE_MASK << 1);
 183
 184                 while (start < end) {
 185                         int idx;
 186
 187                         write_c0_entryhi(start);
 188                         start += (PAGE_SIZE << 1);
 189                         mtc0_tlbw_hazard();
 190                         tlb_probe();
 191                         tlb_probe_hazard();
 192                         idx = read_c0_index();
 193                         write_c0_entrylo0(0);
 194                         write_c0_entrylo1(0);
 195                         if (idx < 0)
 196                                 continue;
 197                         /* Make sure all entries differ. */
 198                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 199                         mtc0_tlbw_hazard();
 200                         tlb_write_indexed();
 201                 }
 202                 tlbw_use_hazard();
 203                 write_c0_entryhi(pid);
 204         } else {
 205                 local_flush_tlb_all();
 206         }
 207         FLUSH_ITLB;
 208         EXIT_CRITICAL(flags);
 209 }
 210
 211 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 212 {
 213         int cpu = smp_processor_id();
 214
 215         if (cpu_context(cpu, vma->vm_mm) != 0) {
 216                 unsigned long flags;
 217                 int oldpid, newpid, idx;
 218
 219                 newpid = cpu_asid(cpu, vma->vm_mm);
 220                 page &= (PAGE_MASK << 1);
 221                 ENTER_CRITICAL(flags);
 222                 oldpid = read_c0_entryhi();
 223                 write_c0_entryhi(page | newpid);
 224                 mtc0_tlbw_hazard();
 225                 tlb_probe();
 226                 tlb_probe_hazard();
 227                 idx = read_c0_index();
 228                 write_c0_entrylo0(0);
 229                 write_c0_entrylo1(0);
 230                 if (idx < 0)
 231                         goto finish;
 232                 /* Make sure all entries differ. */
 233                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 234                 mtc0_tlbw_hazard();
 235                 tlb_write_indexed();
 236                 tlbw_use_hazard();
 237
 238         finish:
 239                 write_c0_entryhi(oldpid);
 240                 FLUSH_ITLB_VM(vma);
 241                 EXIT_CRITICAL(flags);
 242         }
 243 }
 244
 245 /*
 246  * This one is only used for pages with the global bit set so we don't care
 247  * much about the ASID.
 248  */
 249 void local_flush_tlb_one(unsigned long page)
 250 {
 251         unsigned long flags;
 252         int oldpid, idx;
 253
 254         ENTER_CRITICAL(flags);
 255         oldpid = read_c0_entryhi();
 256         page &= (PAGE_MASK << 1);
 257         write_c0_entryhi(page);
 258         mtc0_tlbw_hazard();
 259         tlb_probe();
 260         tlb_probe_hazard();
 261         idx = read_c0_index();
 262         write_c0_entrylo0(0);
 263         write_c0_entrylo1(0);
 264         if (idx >= 0) {
 265                 /* Make sure all entries differ. */
 266                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 267                 mtc0_tlbw_hazard();
 268                 tlb_write_indexed();
 269                 tlbw_use_hazard();
 270         }
 271         write_c0_entryhi(oldpid);
 272         FLUSH_ITLB;
 273         EXIT_CRITICAL(flags);
 274 }
 275
 276 /*
 277  * We will need multiple versions of update_mmu_cache(), one that just
 278  * updates the TLB with the new pte(s), and another which also checks
 279  * for the R4k "end of page" hardware bug and does the needy.
 280  */
 281 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 282 {
 283         unsigned long flags;
 284         pgd_t *pgdp;
 285         pud_t *pudp;
 286         pmd_t *pmdp;
 287         pte_t *ptep;
 288         int idx, pid;
 289
 290         /*
 291          * Handle debugger faulting in for debugee.
 292          */
 293         if (current->active_mm != vma->vm_mm)
 294                 return;
 295
 296         ENTER_CRITICAL(flags);
 297
 298         pid = read_c0_entryhi() & ASID_MASK;
 299         address &= (PAGE_MASK << 1);
 300         write_c0_entryhi(address | pid);
 301         pgdp = pgd_offset(vma->vm_mm, address);
 302         mtc0_tlbw_hazard();
 303         tlb_probe();
 304         tlb_probe_hazard();
 305         pudp = pud_offset(pgdp, address);
 306         pmdp = pmd_offset(pudp, address);
 307         idx = read_c0_index();
 308 #ifdef CONFIG_HUGETLB_PAGE
 309         /* this could be a huge page  */
 310         if (pmd_huge(*pmdp)) {
 311                 unsigned long lo;
 312                 write_c0_pagemask(PM_HUGE_MASK);
 313                 ptep = (pte_t *)pmdp;
 314                 lo = pte_to_entrylo(pte_val(*ptep));
 315                 write_c0_entrylo0(lo);
 316                 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
 317
 318                 mtc0_tlbw_hazard();
 319                 if (idx < 0)
 320                         tlb_write_random();
 321                 else
 322                         tlb_write_indexed();
 323                 tlbw_use_hazard();
 324                 write_c0_pagemask(PM_DEFAULT_MASK);
 325         } else
 326 #endif
 327         {
 328                 ptep = pte_offset_map(pmdp, address);
 329
 330 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
 331                 write_c0_entrylo0(ptep->pte_high);
 332                 ptep++;
 333                 write_c0_entrylo1(ptep->pte_high);
 334 #else
 335                 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
 336                 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
 337 #endif
 338                 mtc0_tlbw_hazard();
 339                 if (idx < 0)
 340                         tlb_write_random();
 341                 else
 342                         tlb_write_indexed();
 343         }
 344         tlbw_use_hazard();
 345         FLUSH_ITLB_VM(vma);
 346         EXIT_CRITICAL(flags);
 347 }
 348
 349 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 350                      unsigned long entryhi, unsigned long pagemask)
 351 {
 352         unsigned long flags;
 353         unsigned long wired;
 354         unsigned long old_pagemask;
 355         unsigned long old_ctx;
 356
 357         ENTER_CRITICAL(flags);
 358         /* Save old context and create impossible VPN2 value */
 359         old_ctx = read_c0_entryhi();
 360         old_pagemask = read_c0_pagemask();
 361         wired = read_c0_wired();
 362         write_c0_wired(wired + 1);
 363         write_c0_index(wired);
 364         tlbw_use_hazard();      /* What is the hazard here? */
 365         write_c0_pagemask(pagemask);
 366         write_c0_entryhi(entryhi);
 367         write_c0_entrylo0(entrylo0);
 368         write_c0_entrylo1(entrylo1);
 369         mtc0_tlbw_hazard();
 370         tlb_write_indexed();
 371         tlbw_use_hazard();
 372
 373         write_c0_entryhi(old_ctx);
 374         tlbw_use_hazard();      /* What is the hazard here? */
 375         write_c0_pagemask(old_pagemask);
 376         local_flush_tlb_all();
 377         EXIT_CRITICAL(flags);
 378 }
 379
 380 static int __cpuinitdata ntlb;
 381 static int __init set_ntlb(char *str)
 382 {
 383         get_option(&str, &ntlb);
 384         return 1;
 385 }
 386
 387 __setup("ntlb=", set_ntlb);
 388
 389 void __cpuinit tlb_init(void)
 390 {
 391         /*
 392          * You should never change this register:
 393          *   - On R4600 1.7 the tlbp never hits for pages smaller than
 394          *     the value in the c0_pagemask register.
 395          *   - The entire mm handling assumes the c0_pagemask register to
 396          *     be set to fixed-size pages.
 397          */
 398         write_c0_pagemask(PM_DEFAULT_MASK);
 399         write_c0_wired(0);
 400         if (current_cpu_type() == CPU_R10000 ||
 401             current_cpu_type() == CPU_R12000 ||
 402             current_cpu_type() == CPU_R14000)
 403                 write_c0_framemask(0);
 404
 405         if (cpu_has_rixi) {
 406                 /*
 407                  * Enable the no read, no exec bits, and enable large virtual
 408                  * address.
 409                  */
 410                 u32 pg = PG_RIE | PG_XIE;
 411 #ifdef CONFIG_64BIT
 412                 pg |= PG_ELPA;
 413 #endif
 414                 write_c0_pagegrain(pg);
 415         }
 416
 417         /* From this point on the ARC firmware is dead.  */
 418         local_flush_tlb_all();
 419
 420         /* Did I tell you that ARC SUCKS?  */
 421
 422         if (ntlb) {
 423                 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
 424                         int wired = current_cpu_data.tlbsize - ntlb;
 425                         write_c0_wired(wired);
 426                         write_c0_index(wired-1);
 427                         printk("Restricting TLB to %d entries\n", ntlb);
 428                 } else
 429                         printk("Ignoring invalid argument ntlb=%d\n", ntlb);
 430         }
 431
 432         build_tlb_refill_handler();
 433 }