1 #ifndef _ASM_X86_MMU_CONTEXT_H
2 #define _ASM_X86_MMU_CONTEXT_H
5 #include <linux/atomic.h>
6 #include <linux/mm_types.h>
8 #include <trace/events/tlb.h>
10 #include <asm/pgalloc.h>
11 #include <asm/tlbflush.h>
12 #include <asm/paravirt.h>
14 #ifndef CONFIG_PARAVIRT
15 static inline void paravirt_activate_mm(struct mm_struct
*prev
,
16 struct mm_struct
*next
)
19 #endif /* !CONFIG_PARAVIRT */
21 #ifdef CONFIG_PERF_EVENTS
22 extern struct static_key rdpmc_always_available
;
24 static inline void load_mm_cr4(struct mm_struct
*mm
)
26 if (static_key_false(&rdpmc_always_available
) ||
27 atomic_read(&mm
->context
.perf_rdpmc_allowed
))
28 cr4_set_bits(X86_CR4_PCE
);
30 cr4_clear_bits(X86_CR4_PCE
);
33 static inline void load_mm_cr4(struct mm_struct
*mm
) {}
36 #ifdef CONFIG_MODIFY_LDT_SYSCALL
38 * ldt_structs can be allocated, used, and freed, but they are never
39 * modified while live.
43 * Xen requires page-aligned LDTs with special permissions. This is
44 * needed to prevent us from installing evil descriptors such as
45 * call gates. On native, we could merge the ldt_struct and LDT
46 * allocations, but it's not worth trying to optimize.
48 struct desc_struct
*entries
;
53 * Used for LDT copy/destruction.
55 int init_new_context(struct task_struct
*tsk
, struct mm_struct
*mm
);
56 void destroy_context(struct mm_struct
*mm
);
57 #else /* CONFIG_MODIFY_LDT_SYSCALL */
58 static inline int init_new_context(struct task_struct
*tsk
,
63 static inline void destroy_context(struct mm_struct
*mm
) {}
66 static inline void load_mm_ldt(struct mm_struct
*mm
)
68 #ifdef CONFIG_MODIFY_LDT_SYSCALL
69 struct ldt_struct
*ldt
;
71 /* lockless_dereference synchronizes with smp_store_release */
72 ldt
= lockless_dereference(mm
->context
.ldt
);
75 * Any change to mm->context.ldt is followed by an IPI to all
76 * CPUs with the mm active. The LDT will not be freed until
77 * after the IPI is handled by all such CPUs. This means that,
78 * if the ldt_struct changes before we return, the values we see
79 * will be safe, and the new values will be loaded before we run
82 * NB: don't try to convert this to use RCU without extreme care.
83 * We would still need IRQs off, because we don't want to change
84 * the local LDT after an IPI loaded a newer value than the one
89 set_ldt(ldt
->entries
, ldt
->size
);
96 DEBUG_LOCKS_WARN_ON(preemptible());
99 static inline void enter_lazy_tlb(struct mm_struct
*mm
, struct task_struct
*tsk
)
102 if (this_cpu_read(cpu_tlbstate
.state
) == TLBSTATE_OK
)
103 this_cpu_write(cpu_tlbstate
.state
, TLBSTATE_LAZY
);
107 static inline void switch_mm(struct mm_struct
*prev
, struct mm_struct
*next
,
108 struct task_struct
*tsk
)
110 unsigned cpu
= smp_processor_id();
112 if (likely(prev
!= next
)) {
114 this_cpu_write(cpu_tlbstate
.state
, TLBSTATE_OK
);
115 this_cpu_write(cpu_tlbstate
.active_mm
, next
);
117 cpumask_set_cpu(cpu
, mm_cpumask(next
));
120 * Re-load page tables.
122 * This logic has an ordering constraint:
124 * CPU 0: Write to a PTE for 'next'
125 * CPU 0: load bit 1 in mm_cpumask. if nonzero, send IPI.
126 * CPU 1: set bit 1 in next's mm_cpumask
127 * CPU 1: load from the PTE that CPU 0 writes (implicit)
129 * We need to prevent an outcome in which CPU 1 observes
130 * the new PTE value and CPU 0 observes bit 1 clear in
131 * mm_cpumask. (If that occurs, then the IPI will never
132 * be sent, and CPU 0's TLB will contain a stale entry.)
134 * The bad outcome can occur if either CPU's load is
135 * reordered before that CPU's store, so both CPUs must
136 * execute full barriers to prevent this from happening.
138 * Thus, switch_mm needs a full barrier between the
139 * store to mm_cpumask and any operation that could load
140 * from next->pgd. TLB fills are special and can happen
141 * due to instruction fetches or for no reason at all,
142 * and neither LOCK nor MFENCE orders them.
143 * Fortunately, load_cr3() is serializing and gives the
144 * ordering guarantee we need.
149 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH
, TLB_FLUSH_ALL
);
151 /* Stop flush ipis for the previous mm */
152 cpumask_clear_cpu(cpu
, mm_cpumask(prev
));
154 /* Load per-mm CR4 state */
157 #ifdef CONFIG_MODIFY_LDT_SYSCALL
159 * Load the LDT, if the LDT is different.
161 * It's possible that prev->context.ldt doesn't match
162 * the LDT register. This can happen if leave_mm(prev)
163 * was called and then modify_ldt changed
164 * prev->context.ldt but suppressed an IPI to this CPU.
165 * In this case, prev->context.ldt != NULL, because we
166 * never set context.ldt to NULL while the mm still
167 * exists. That means that next->context.ldt !=
168 * prev->context.ldt, because mms never share an LDT.
170 if (unlikely(prev
->context
.ldt
!= next
->context
.ldt
))
176 this_cpu_write(cpu_tlbstate
.state
, TLBSTATE_OK
);
177 BUG_ON(this_cpu_read(cpu_tlbstate
.active_mm
) != next
);
179 if (!cpumask_test_cpu(cpu
, mm_cpumask(next
))) {
181 * On established mms, the mm_cpumask is only changed
182 * from irq context, from ptep_clear_flush() while in
183 * lazy tlb mode, and here. Irqs are blocked during
184 * schedule, protecting us from simultaneous changes.
186 cpumask_set_cpu(cpu
, mm_cpumask(next
));
189 * We were in lazy tlb mode and leave_mm disabled
190 * tlb flush IPI delivery. We must reload CR3
191 * to make sure to use no freed page tables.
193 * As above, load_cr3() is serializing and orders TLB
194 * fills with respect to the mm_cpumask write.
197 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH
, TLB_FLUSH_ALL
);
205 #define activate_mm(prev, next) \
207 paravirt_activate_mm((prev), (next)); \
208 switch_mm((prev), (next), NULL); \
212 #define deactivate_mm(tsk, mm) \
217 #define deactivate_mm(tsk, mm) \
220 loadsegment(fs, 0); \
224 static inline void arch_dup_mmap(struct mm_struct
*oldmm
,
225 struct mm_struct
*mm
)
227 paravirt_arch_dup_mmap(oldmm
, mm
);
230 static inline void arch_exit_mmap(struct mm_struct
*mm
)
232 paravirt_arch_exit_mmap(mm
);
236 static inline bool is_64bit_mm(struct mm_struct
*mm
)
238 return !config_enabled(CONFIG_IA32_EMULATION
) ||
239 !(mm
->context
.ia32_compat
== TIF_IA32
);
242 static inline bool is_64bit_mm(struct mm_struct
*mm
)
248 static inline void arch_bprm_mm_init(struct mm_struct
*mm
,
249 struct vm_area_struct
*vma
)
254 static inline void arch_unmap(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
255 unsigned long start
, unsigned long end
)
258 * mpx_notify_unmap() goes and reads a rarely-hot
259 * cacheline in the mm_struct. That can be expensive
260 * enough to be seen in profiles.
262 * The mpx_notify_unmap() call and its contents have been
263 * observed to affect munmap() performance on hardware
264 * where MPX is not present.
266 * The unlikely() optimizes for the fast case: no MPX
267 * in the CPU, or no MPX use in the process. Even if
268 * we get this wrong (in the unlikely event that MPX
269 * is widely enabled on some system) the overhead of
270 * MPX itself (reading bounds tables) is expected to
271 * overwhelm the overhead of getting this unlikely()
272 * consistently wrong.
274 if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX
)))
275 mpx_notify_unmap(mm
, vma
, start
, end
);
278 static inline int vma_pkey(struct vm_area_struct
*vma
)
281 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
282 unsigned long vma_pkey_mask
= VM_PKEY_BIT0
| VM_PKEY_BIT1
|
283 VM_PKEY_BIT2
| VM_PKEY_BIT3
;
284 pkey
= (vma
->vm_flags
& vma_pkey_mask
) >> VM_PKEY_SHIFT
;
289 static inline bool __pkru_allows_pkey(u16 pkey
, bool write
)
291 u32 pkru
= read_pkru();
293 if (!__pkru_allows_read(pkru
, pkey
))
295 if (write
&& !__pkru_allows_write(pkru
, pkey
))
302 * We only want to enforce protection keys on the current process
303 * because we effectively have no access to PKRU for other
304 * processes or any way to tell *which * PKRU in a threaded
305 * process we could use.
307 * So do not enforce things if the VMA is not from the current
308 * mm, or if we are in a kernel thread.
310 static inline bool vma_is_foreign(struct vm_area_struct
*vma
)
315 * Should PKRU be enforced on the access to this VMA? If
316 * the VMA is from another process, then PKRU has no
317 * relevance and should not be enforced.
319 if (current
->mm
!= vma
->vm_mm
)
325 static inline bool arch_vma_access_permitted(struct vm_area_struct
*vma
,
326 bool write
, bool foreign
)
328 /* allow access if the VMA is not one from this process */
329 if (foreign
|| vma_is_foreign(vma
))
331 return __pkru_allows_pkey(vma_pkey(vma
), write
);
334 static inline bool arch_pte_access_permitted(pte_t pte
, bool write
)
336 return __pkru_allows_pkey(pte_flags_pkey(pte_flags(pte
)), write
);
339 #endif /* _ASM_X86_MMU_CONTEXT_H */