arch/powerpc/mm/mmu_context_nohash.c

   1 /*
   2  * This file contains the routines for handling the MMU on those
   3  * PowerPC implementations where the MMU is not using the hash
   4  * table, such as 8xx, 4xx, BookE's etc...
   5  *
   6  * Copyright 2008 Ben Herrenschmidt <benh@kernel.crashing.org>
   7  *                IBM Corp.
   8  *
   9  *  Derived from previous arch/powerpc/mm/mmu_context.c
  10  *  and arch/powerpc/include/asm/mmu_context.h
  11  *
  12  *  This program is free software; you can redistribute it and/or
  13  *  modify it under the terms of the GNU General Public License
  14  *  as published by the Free Software Foundation; either version
  15  *  2 of the License, or (at your option) any later version.
  16  *
  17  * TODO:
  18  *
  19  *   - The global context lock will not scale very well
  20  *   - The maps should be dynamically allocated to allow for processors
  21  *     that support more PID bits at runtime
  22  *   - Implement flush_tlb_mm() by making the context stale and picking
  23  *     a new one
  24  *   - More aggressively clear stale map bits and maybe find some way to
  25  *     also clear mm->cpu_vm_mask bits when processes are migrated
  26  */
  27
  28 #undef DEBUG
  29 #define DEBUG_STEAL_ONLY
  30 #undef DEBUG_MAP_CONSISTENCY
  31 /*#define DEBUG_CLAMP_LAST_CONTEXT   15 */
  32
  33 #include <linux/kernel.h>
  34 #include <linux/mm.h>
  35 #include <linux/init.h>
  36 #include <linux/spinlock.h>
  37 #include <linux/bootmem.h>
  38 #include <linux/notifier.h>
  39 #include <linux/cpu.h>
  40
  41 #include <asm/mmu_context.h>
  42 #include <asm/tlbflush.h>
  43
  44 static unsigned int first_context, last_context;
  45 static unsigned int next_context, nr_free_contexts;
  46 static unsigned long *context_map;
  47 static unsigned long *stale_map[NR_CPUS];
  48 static struct mm_struct **context_mm;
  49 static DEFINE_SPINLOCK(context_lock);
  50
  51 #define CTX_MAP_SIZE    \
  52         (sizeof(unsigned long) * (last_context / BITS_PER_LONG + 1))
  53
  54
  55 /* Steal a context from a task that has one at the moment.
  56  *
  57  * This is used when we are running out of available PID numbers
  58  * on the processors.
  59  *
  60  * This isn't an LRU system, it just frees up each context in
  61  * turn (sort-of pseudo-random replacement :).  This would be the
  62  * place to implement an LRU scheme if anyone was motivated to do it.
  63  *  -- paulus
  64  *
  65  * For context stealing, we use a slightly different approach for
  66  * SMP and UP. Basically, the UP one is simpler and doesn't use
  67  * the stale map as we can just flush the local CPU
  68  *  -- benh
  69  */
  70 #ifdef CONFIG_SMP
  71 static unsigned int steal_context_smp(unsigned int id)
  72 {
  73         struct mm_struct *mm;
  74         unsigned int cpu, max;
  75
  76         max = last_context - first_context;
  77
  78         /* Attempt to free next_context first and then loop until we manage */
  79         while (max--) {
  80                 /* Pick up the victim mm */
  81                 mm = context_mm[id];
  82
  83                 /* We have a candidate victim, check if it's active, on SMP
  84                  * we cannot steal active contexts
  85                  */
  86                 if (mm->context.active) {
  87                         id++;
  88                         if (id > last_context)
  89                                 id = first_context;
  90                         continue;
  91                 }
  92                 pr_debug("[%d] steal context %d from mm @%p\n",
  93                          smp_processor_id(), id, mm);
  94
  95                 /* Mark this mm has having no context anymore */
  96                 mm->context.id = MMU_NO_CONTEXT;
  97
  98                 /* Mark it stale on all CPUs that used this mm */
  99                 for_each_cpu(cpu, mm_cpumask(mm))
 100                         __set_bit(id, stale_map[cpu]);
 101                 return id;
 102         }
 103
 104         /* This will happen if you have more CPUs than available contexts,
 105          * all we can do here is wait a bit and try again
 106          */
 107         spin_unlock(&context_lock);
 108         cpu_relax();
 109         spin_lock(&context_lock);
 110
 111         /* This will cause the caller to try again */
 112         return MMU_NO_CONTEXT;
 113 }
 114 #endif  /* CONFIG_SMP */
 115
 116 /* Note that this will also be called on SMP if all other CPUs are
 117  * offlined, which means that it may be called for cpu != 0. For
 118  * this to work, we somewhat assume that CPUs that are onlined
 119  * come up with a fully clean TLB (or are cleaned when offlined)
 120  */
 121 static unsigned int steal_context_up(unsigned int id)
 122 {
 123         struct mm_struct *mm;
 124         int cpu = smp_processor_id();
 125
 126         /* Pick up the victim mm */
 127         mm = context_mm[id];
 128
 129         pr_debug("[%d] steal context %d from mm @%p\n", cpu, id, mm);
 130
 131         /* Flush the TLB for that context */
 132         local_flush_tlb_mm(mm);
 133
 134         /* Mark this mm has having no context anymore */
 135         mm->context.id = MMU_NO_CONTEXT;
 136
 137         /* XXX This clear should ultimately be part of local_flush_tlb_mm */
 138         __clear_bit(id, stale_map[cpu]);
 139
 140         return id;
 141 }
 142
 143 #ifdef DEBUG_MAP_CONSISTENCY
 144 static void context_check_map(void)
 145 {
 146         unsigned int id, nrf, nact;
 147
 148         nrf = nact = 0;
 149         for (id = first_context; id <= last_context; id++) {
 150                 int used = test_bit(id, context_map);
 151                 if (!used)
 152                         nrf++;
 153                 if (used != (context_mm[id] != NULL))
 154                         pr_err("MMU: Context %d is %s and MM is %p !\n",
 155                                id, used ? "used" : "free", context_mm[id]);
 156                 if (context_mm[id] != NULL)
 157                         nact += context_mm[id]->context.active;
 158         }
 159         if (nrf != nr_free_contexts) {
 160                 pr_err("MMU: Free context count out of sync ! (%d vs %d)\n",
 161                        nr_free_contexts, nrf);
 162                 nr_free_contexts = nrf;
 163         }
 164         if (nact > num_online_cpus())
 165                 pr_err("MMU: More active contexts than CPUs ! (%d vs %d)\n",
 166                        nact, num_online_cpus());
 167         if (first_context > 0 && !test_bit(0, context_map))
 168                 pr_err("MMU: Context 0 has been freed !!!\n");
 169 }
 170 #else
 171 static void context_check_map(void) { }
 172 #endif
 173
 174 void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
 175 {
 176         unsigned int id, cpu = smp_processor_id();
 177         unsigned long *map;
 178
 179         /* No lockless fast path .. yet */
 180         spin_lock(&context_lock);
 181
 182 #ifndef DEBUG_STEAL_ONLY
 183         pr_debug("[%d] activating context for mm @%p, active=%d, id=%d\n",
 184                  cpu, next, next->context.active, next->context.id);
 185 #endif
 186
 187 #ifdef CONFIG_SMP
 188         /* Mark us active and the previous one not anymore */
 189         next->context.active++;
 190         if (prev) {
 191 #ifndef DEBUG_STEAL_ONLY
 192                 pr_debug(" old context %p active was: %d\n",
 193                          prev, prev->context.active);
 194 #endif
 195                 WARN_ON(prev->context.active < 1);
 196                 prev->context.active--;
 197         }
 198
 199  again:
 200 #endif /* CONFIG_SMP */
 201
 202         /* If we already have a valid assigned context, skip all that */
 203         id = next->context.id;
 204         if (likely(id != MMU_NO_CONTEXT))
 205                 goto ctxt_ok;
 206
 207         /* We really don't have a context, let's try to acquire one */
 208         id = next_context;
 209         if (id > last_context)
 210                 id = first_context;
 211         map = context_map;
 212
 213         /* No more free contexts, let's try to steal one */
 214         if (nr_free_contexts == 0) {
 215 #ifdef CONFIG_SMP
 216                 if (num_online_cpus() > 1) {
 217                         id = steal_context_smp(id);
 218                         if (id == MMU_NO_CONTEXT)
 219                                 goto again;
 220                 }
 221 #endif /* CONFIG_SMP */
 222                 id = steal_context_up(id);
 223                 goto stolen;
 224         }
 225         nr_free_contexts--;
 226
 227         /* We know there's at least one free context, try to find it */
 228         while (__test_and_set_bit(id, map)) {
 229                 id = find_next_zero_bit(map, last_context+1, id);
 230                 if (id > last_context)
 231                         id = first_context;
 232         }
 233  stolen:
 234         next_context = id + 1;
 235         context_mm[id] = next;
 236         next->context.id = id;
 237
 238 #ifndef DEBUG_STEAL_ONLY
 239         pr_debug("[%d] picked up new id %d, nrf is now %d\n",
 240                  cpu, id, nr_free_contexts);
 241 #endif
 242
 243         context_check_map();
 244  ctxt_ok:
 245
 246         /* If that context got marked stale on this CPU, then flush the
 247          * local TLB for it and unmark it before we use it
 248          */
 249         if (test_bit(id, stale_map[cpu])) {
 250                 pr_debug("[%d] flushing stale context %d for mm @%p !\n",
 251                          cpu, id, next);
 252                 local_flush_tlb_mm(next);
 253
 254                 /* XXX This clear should ultimately be part of local_flush_tlb_mm */
 255                 __clear_bit(id, stale_map[cpu]);
 256         }
 257
 258         /* Flick the MMU and release lock */
 259         set_context(id, next->pgd);
 260         spin_unlock(&context_lock);
 261 }
 262
 263 /*
 264  * Set up the context for a new address space.
 265  */
 266 int init_new_context(struct task_struct *t, struct mm_struct *mm)
 267 {
 268         mm->context.id = MMU_NO_CONTEXT;
 269         mm->context.active = 0;
 270
 271         return 0;
 272 }
 273
 274 /*
 275  * We're finished using the context for an address space.
 276  */
 277 void destroy_context(struct mm_struct *mm)
 278 {
 279         unsigned long flags;
 280         unsigned int id;
 281
 282         if (mm->context.id == MMU_NO_CONTEXT)
 283                 return;
 284
 285         WARN_ON(mm->context.active != 0);
 286
 287         spin_lock_irqsave(&context_lock, flags);
 288         id = mm->context.id;
 289         if (id != MMU_NO_CONTEXT) {
 290                 __clear_bit(id, context_map);
 291                 mm->context.id = MMU_NO_CONTEXT;
 292 #ifdef DEBUG_MAP_CONSISTENCY
 293                 mm->context.active = 0;
 294 #endif
 295                 context_mm[id] = NULL;
 296                 nr_free_contexts++;
 297         }
 298         spin_unlock_irqrestore(&context_lock, flags);
 299 }
 300
 301 #ifdef CONFIG_SMP
 302
 303 static int __cpuinit mmu_context_cpu_notify(struct notifier_block *self,
 304                                             unsigned long action, void *hcpu)
 305 {
 306         unsigned int cpu = (unsigned int)(long)hcpu;
 307
 308         /* We don't touch CPU 0 map, it's allocated at aboot and kept
 309          * around forever
 310          */
 311         if (cpu == 0)
 312                 return NOTIFY_OK;
 313
 314         switch (action) {
 315         case CPU_ONLINE:
 316         case CPU_ONLINE_FROZEN:
 317                 pr_debug("MMU: Allocating stale context map for CPU %d\n", cpu);
 318                 stale_map[cpu] = kzalloc(CTX_MAP_SIZE, GFP_KERNEL);
 319                 break;
 320 #ifdef CONFIG_HOTPLUG_CPU
 321         case CPU_DEAD:
 322         case CPU_DEAD_FROZEN:
 323                 pr_debug("MMU: Freeing stale context map for CPU %d\n", cpu);
 324                 kfree(stale_map[cpu]);
 325                 stale_map[cpu] = NULL;
 326                 break;
 327 #endif
 328         }
 329         return NOTIFY_OK;
 330 }
 331
 332 static struct notifier_block __cpuinitdata mmu_context_cpu_nb = {
 333         .notifier_call  = mmu_context_cpu_notify,
 334 };
 335
 336 #endif /* CONFIG_SMP */
 337
 338 /*
 339  * Initialize the context management stuff.
 340  */
 341 void __init mmu_context_init(void)
 342 {
 343         /* Mark init_mm as being active on all possible CPUs since
 344          * we'll get called with prev == init_mm the first time
 345          * we schedule on a given CPU
 346          */
 347         init_mm.context.active = NR_CPUS;
 348
 349         /*
 350          *   The MPC8xx has only 16 contexts.  We rotate through them on each
 351          * task switch.  A better way would be to keep track of tasks that
 352          * own contexts, and implement an LRU usage.  That way very active
 353          * tasks don't always have to pay the TLB reload overhead.  The
 354          * kernel pages are mapped shared, so the kernel can run on behalf
 355          * of any task that makes a kernel entry.  Shared does not mean they
 356          * are not protected, just that the ASID comparison is not performed.
 357          *      -- Dan
 358          *
 359          * The IBM4xx has 256 contexts, so we can just rotate through these
 360          * as a way of "switching" contexts.  If the TID of the TLB is zero,
 361          * the PID/TID comparison is disabled, so we can use a TID of zero
 362          * to represent all kernel pages as shared among all contexts.
 363          *      -- Dan
 364          */
 365         if (mmu_has_feature(MMU_FTR_TYPE_8xx)) {
 366                 first_context = 0;
 367                 last_context = 15;
 368         } else {
 369                 first_context = 1;
 370                 last_context = 255;
 371         }
 372
 373 #ifdef DEBUG_CLAMP_LAST_CONTEXT
 374         last_context = DEBUG_CLAMP_LAST_CONTEXT;
 375 #endif
 376         /*
 377          * Allocate the maps used by context management
 378          */
 379         context_map = alloc_bootmem(CTX_MAP_SIZE);
 380         context_mm = alloc_bootmem(sizeof(void *) * (last_context + 1));
 381         stale_map[0] = alloc_bootmem(CTX_MAP_SIZE);
 382
 383 #ifdef CONFIG_SMP
 384         register_cpu_notifier(&mmu_context_cpu_nb);
 385 #endif
 386
 387         printk(KERN_INFO
 388                "MMU: Allocated %zu bytes of context maps for %d contexts\n",
 389                2 * CTX_MAP_SIZE + (sizeof(void *) * (last_context + 1)),
 390                last_context - first_context + 1);
 391
 392         /*
 393          * Some processors have too few contexts to reserve one for
 394          * init_mm, and require using context 0 for a normal task.
 395          * Other processors reserve the use of context zero for the kernel.
 396          * This code assumes first_context < 32.
 397          */
 398         context_map[0] = (1 << first_context) - 1;
 399         next_context = first_context;
 400         nr_free_contexts = last_context - first_context + 1;
 401 }
 402