arch/x86/include/asm/mmu_context.h

   1 #ifndef _ASM_X86_MMU_CONTEXT_H
   2 #define _ASM_X86_MMU_CONTEXT_H
   3
   4 #include <asm/desc.h>
   5 #include <linux/atomic.h>
   6 #include <linux/mm_types.h>
   7
   8 #include <trace/events/tlb.h>
   9
  10 #include <asm/pgalloc.h>
  11 #include <asm/tlbflush.h>
  12 #include <asm/paravirt.h>
  13 #ifndef CONFIG_PARAVIRT
  14 #include <asm-generic/mm_hooks.h>
  15
  16 static inline void paravirt_activate_mm(struct mm_struct *prev,
  17                                         struct mm_struct *next)
  18 {
  19 }
  20 #endif  /* !CONFIG_PARAVIRT */
  21
  22 /*
  23  * Used for LDT copy/destruction.
  24  */
  25 int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
  26 void destroy_context(struct mm_struct *mm);
  27
  28
  29 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
  30 {
  31 #ifdef CONFIG_SMP
  32         if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
  33                 this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
  34 #endif
  35 }
  36
  37 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
  38                              struct task_struct *tsk)
  39 {
  40         unsigned cpu = smp_processor_id();
  41
  42         if (likely(prev != next)) {
  43 #ifdef CONFIG_SMP
  44                 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
  45                 this_cpu_write(cpu_tlbstate.active_mm, next);
  46 #endif
  47                 cpumask_set_cpu(cpu, mm_cpumask(next));
  48
  49                 /* Re-load page tables */
  50                 load_cr3(next->pgd);
  51                 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
  52
  53                 /* Stop flush ipis for the previous mm */
  54                 cpumask_clear_cpu(cpu, mm_cpumask(prev));
  55
  56                 /* Load the LDT, if the LDT is different: */
  57                 if (unlikely(prev->context.ldt != next->context.ldt))
  58                         load_LDT_nolock(&next->context);
  59         }
  60 #ifdef CONFIG_SMP
  61           else {
  62                 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
  63                 BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
  64
  65                 if (!cpumask_test_cpu(cpu, mm_cpumask(next))) {
  66                         /*
  67                          * On established mms, the mm_cpumask is only changed
  68                          * from irq context, from ptep_clear_flush() while in
  69                          * lazy tlb mode, and here. Irqs are blocked during
  70                          * schedule, protecting us from simultaneous changes.
  71                          */
  72                         cpumask_set_cpu(cpu, mm_cpumask(next));
  73                         /*
  74                          * We were in lazy tlb mode and leave_mm disabled
  75                          * tlb flush IPI delivery. We must reload CR3
  76                          * to make sure to use no freed page tables.
  77                          */
  78                         load_cr3(next->pgd);
  79                         trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
  80                         load_LDT_nolock(&next->context);
  81                 }
  82         }
  83 #endif
  84 }
  85
  86 #define activate_mm(prev, next)                 \
  87 do {                                            \
  88         paravirt_activate_mm((prev), (next));   \
  89         switch_mm((prev), (next), NULL);        \
  90 } while (0);
  91
  92 #ifdef CONFIG_X86_32
  93 #define deactivate_mm(tsk, mm)                  \
  94 do {                                            \
  95         lazy_load_gs(0);                        \
  96 } while (0)
  97 #else
  98 #define deactivate_mm(tsk, mm)                  \
  99 do {                                            \
 100         load_gs_index(0);                       \
 101         loadsegment(fs, 0);                     \
 102 } while (0)
 103 #endif
 104
 105 #endif /* _ASM_X86_MMU_CONTEXT_H */