arch/x86/include/asm/tlbflush.h

   1 #ifndef _ASM_X86_TLBFLUSH_H
   2 #define _ASM_X86_TLBFLUSH_H
   3
   4 #include <linux/mm.h>
   5 #include <linux/sched.h>
   6
   7 #include <asm/processor.h>
   8 #include <asm/special_insns.h>
   9
  10 static inline void __invpcid(unsigned long pcid, unsigned long addr,
  11                              unsigned long type)
  12 {
  13         u64 desc[2] = { pcid, addr };
  14
  15         /*
  16          * The memory clobber is because the whole point is to invalidate
  17          * stale TLB entries and, especially if we're flushing global
  18          * mappings, we don't want the compiler to reorder any subsequent
  19          * memory accesses before the TLB flush.
  20          *
  21          * The hex opcode is invpcid (%ecx), %eax in 32-bit mode and
  22          * invpcid (%rcx), %rax in long mode.
  23          */
  24         asm volatile (".byte 0x66, 0x0f, 0x38, 0x82, 0x01"
  25                       : : "m" (desc), "a" (type), "c" (desc) : "memory");
  26 }
  27
  28 #define INVPCID_TYPE_INDIV_ADDR         0
  29 #define INVPCID_TYPE_SINGLE_CTXT        1
  30 #define INVPCID_TYPE_ALL_INCL_GLOBAL    2
  31 #define INVPCID_TYPE_ALL_NON_GLOBAL     3
  32
  33 /* Flush all mappings for a given pcid and addr, not including globals. */
  34 static inline void invpcid_flush_one(unsigned long pcid,
  35                                      unsigned long addr)
  36 {
  37         __invpcid(pcid, addr, INVPCID_TYPE_INDIV_ADDR);
  38 }
  39
  40 /* Flush all mappings for a given PCID, not including globals. */
  41 static inline void invpcid_flush_single_context(unsigned long pcid)
  42 {
  43         __invpcid(pcid, 0, INVPCID_TYPE_SINGLE_CTXT);
  44 }
  45
  46 /* Flush all mappings, including globals, for all PCIDs. */
  47 static inline void invpcid_flush_all(void)
  48 {
  49         __invpcid(0, 0, INVPCID_TYPE_ALL_INCL_GLOBAL);
  50 }
  51
  52 /* Flush all mappings for all PCIDs except globals. */
  53 static inline void invpcid_flush_all_nonglobals(void)
  54 {
  55         __invpcid(0, 0, INVPCID_TYPE_ALL_NON_GLOBAL);
  56 }
  57
  58 #ifdef CONFIG_PARAVIRT
  59 #include <asm/paravirt.h>
  60 #else
  61 #define __flush_tlb() __native_flush_tlb()
  62 #define __flush_tlb_global() __native_flush_tlb_global()
  63 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
  64 #endif
  65
  66 struct tlb_state {
  67 #ifdef CONFIG_SMP
  68         struct mm_struct *active_mm;
  69         int state;
  70 #endif
  71
  72         /*
  73          * Access to this CR4 shadow and to H/W CR4 is protected by
  74          * disabling interrupts when modifying either one.
  75          */
  76         unsigned long cr4;
  77 };
  78 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
  79
  80 /* Initialize cr4 shadow for this CPU. */
  81 static inline void cr4_init_shadow(void)
  82 {
  83         this_cpu_write(cpu_tlbstate.cr4, __read_cr4());
  84 }
  85
  86 /* Set in this cpu's CR4. */
  87 static inline void cr4_set_bits(unsigned long mask)
  88 {
  89         unsigned long cr4;
  90
  91         cr4 = this_cpu_read(cpu_tlbstate.cr4);
  92         if ((cr4 | mask) != cr4) {
  93                 cr4 |= mask;
  94                 this_cpu_write(cpu_tlbstate.cr4, cr4);
  95                 __write_cr4(cr4);
  96         }
  97 }
  98
  99 /* Clear in this cpu's CR4. */
 100 static inline void cr4_clear_bits(unsigned long mask)
 101 {
 102         unsigned long cr4;
 103
 104         cr4 = this_cpu_read(cpu_tlbstate.cr4);
 105         if ((cr4 & ~mask) != cr4) {
 106                 cr4 &= ~mask;
 107                 this_cpu_write(cpu_tlbstate.cr4, cr4);
 108                 __write_cr4(cr4);
 109         }
 110 }
 111
 112 /* Read the CR4 shadow. */
 113 static inline unsigned long cr4_read_shadow(void)
 114 {
 115         return this_cpu_read(cpu_tlbstate.cr4);
 116 }
 117
 118 /*
 119  * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
 120  * enable and PPro Global page enable), so that any CPU's that boot
 121  * up after us can get the correct flags.  This should only be used
 122  * during boot on the boot cpu.
 123  */
 124 extern unsigned long mmu_cr4_features;
 125 extern u32 *trampoline_cr4_features;
 126
 127 static inline void cr4_set_bits_and_update_boot(unsigned long mask)
 128 {
 129         mmu_cr4_features |= mask;
 130         if (trampoline_cr4_features)
 131                 *trampoline_cr4_features = mmu_cr4_features;
 132         cr4_set_bits(mask);
 133 }
 134
 135 static inline void __native_flush_tlb(void)
 136 {
 137         native_write_cr3(native_read_cr3());
 138 }
 139
 140 static inline void __native_flush_tlb_global_irq_disabled(void)
 141 {
 142         unsigned long cr4;
 143
 144         cr4 = this_cpu_read(cpu_tlbstate.cr4);
 145         /* clear PGE */
 146         native_write_cr4(cr4 & ~X86_CR4_PGE);
 147         /* write old PGE again and flush TLBs */
 148         native_write_cr4(cr4);
 149 }
 150
 151 static inline void __native_flush_tlb_global(void)
 152 {
 153         unsigned long flags;
 154
 155         /*
 156          * Read-modify-write to CR4 - protect it from preemption and
 157          * from interrupts. (Use the raw variant because this code can
 158          * be called from deep inside debugging code.)
 159          */
 160         raw_local_irq_save(flags);
 161
 162         __native_flush_tlb_global_irq_disabled();
 163
 164         raw_local_irq_restore(flags);
 165 }
 166
 167 static inline void __native_flush_tlb_single(unsigned long addr)
 168 {
 169         asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
 170 }
 171
 172 static inline void __flush_tlb_all(void)
 173 {
 174         if (cpu_has_pge)
 175                 __flush_tlb_global();
 176         else
 177                 __flush_tlb();
 178 }
 179
 180 static inline void __flush_tlb_one(unsigned long addr)
 181 {
 182         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
 183         __flush_tlb_single(addr);
 184 }
 185
 186 #define TLB_FLUSH_ALL   -1UL
 187
 188 /*
 189  * TLB flushing:
 190  *
 191  *  - flush_tlb() flushes the current mm struct TLBs
 192  *  - flush_tlb_all() flushes all processes TLBs
 193  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 194  *  - flush_tlb_page(vma, vmaddr) flushes one page
 195  *  - flush_tlb_range(vma, start, end) flushes a range of pages
 196  *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
 197  *  - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
 198  *
 199  * ..but the i386 has somewhat limited tlb flushing capabilities,
 200  * and page-granular flushes are available only on i486 and up.
 201  */
 202
 203 #ifndef CONFIG_SMP
 204
 205 /* "_up" is for UniProcessor.
 206  *
 207  * This is a helper for other header functions.  *Not* intended to be called
 208  * directly.  All global TLB flushes need to either call this, or to bump the
 209  * vm statistics themselves.
 210  */
 211 static inline void __flush_tlb_up(void)
 212 {
 213         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
 214         __flush_tlb();
 215 }
 216
 217 static inline void flush_tlb_all(void)
 218 {
 219         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
 220         __flush_tlb_all();
 221 }
 222
 223 static inline void flush_tlb(void)
 224 {
 225         __flush_tlb_up();
 226 }
 227
 228 static inline void local_flush_tlb(void)
 229 {
 230         __flush_tlb_up();
 231 }
 232
 233 static inline void flush_tlb_mm(struct mm_struct *mm)
 234 {
 235         if (mm == current->active_mm)
 236                 __flush_tlb_up();
 237 }
 238
 239 static inline void flush_tlb_page(struct vm_area_struct *vma,
 240                                   unsigned long addr)
 241 {
 242         if (vma->vm_mm == current->active_mm)
 243                 __flush_tlb_one(addr);
 244 }
 245
 246 static inline void flush_tlb_range(struct vm_area_struct *vma,
 247                                    unsigned long start, unsigned long end)
 248 {
 249         if (vma->vm_mm == current->active_mm)
 250                 __flush_tlb_up();
 251 }
 252
 253 static inline void flush_tlb_mm_range(struct mm_struct *mm,
 254            unsigned long start, unsigned long end, unsigned long vmflag)
 255 {
 256         if (mm == current->active_mm)
 257                 __flush_tlb_up();
 258 }
 259
 260 static inline void native_flush_tlb_others(const struct cpumask *cpumask,
 261                                            struct mm_struct *mm,
 262                                            unsigned long start,
 263                                            unsigned long end)
 264 {
 265 }
 266
 267 static inline void reset_lazy_tlbstate(void)
 268 {
 269 }
 270
 271 static inline void flush_tlb_kernel_range(unsigned long start,
 272                                           unsigned long end)
 273 {
 274         flush_tlb_all();
 275 }
 276
 277 #else  /* SMP */
 278
 279 #include <asm/smp.h>
 280
 281 #define local_flush_tlb() __flush_tlb()
 282
 283 #define flush_tlb_mm(mm)        flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL)
 284
 285 #define flush_tlb_range(vma, start, end)        \
 286                 flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags)
 287
 288 extern void flush_tlb_all(void);
 289 extern void flush_tlb_current_task(void);
 290 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
 291 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 292                                 unsigned long end, unsigned long vmflag);
 293 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
 294
 295 #define flush_tlb()     flush_tlb_current_task()
 296
 297 void native_flush_tlb_others(const struct cpumask *cpumask,
 298                                 struct mm_struct *mm,
 299                                 unsigned long start, unsigned long end);
 300
 301 #define TLBSTATE_OK     1
 302 #define TLBSTATE_LAZY   2
 303
 304 static inline void reset_lazy_tlbstate(void)
 305 {
 306         this_cpu_write(cpu_tlbstate.state, 0);
 307         this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
 308 }
 309
 310 #endif  /* SMP */
 311
 312 /* Not inlined due to inc_irq_stat not being defined yet */
 313 #define flush_tlb_local() {             \
 314         inc_irq_stat(irq_tlb_count);    \
 315         local_flush_tlb();              \
 316 }
 317
 318 #ifndef CONFIG_PARAVIRT
 319 #define flush_tlb_others(mask, mm, start, end)  \
 320         native_flush_tlb_others(mask, mm, start, end)
 321 #endif
 322
 323 #endif /* _ASM_X86_TLBFLUSH_H */