[deliverable/linux.git] / arch / x86 / include / asm / mmu_context.h

#ifndef _ASM_X86_MMU_CONTEXT_H
#define _ASM_X86_MMU_CONTEXT_H

#include <asm/desc.h>
#include <linux/atomic.h>
#include <linux/mm_types.h>

#include <trace/events/tlb.h>

#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm/paravirt.h>
#ifndef CONFIG_PARAVIRT
#include <asm-generic/mm_hooks.h>

static inline void paravirt_activate_mm(struct mm_struct *prev,
					struct mm_struct *next)
{
}
#endif	/* !CONFIG_PARAVIRT */

/*
 * Used for LDT copy/destruction.
 */
int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
void destroy_context(struct mm_struct *mm);


static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
	if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
		this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}

static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
			     struct task_struct *tsk)
{
	unsigned cpu = smp_processor_id();

	if (likely(prev != next)) {
#ifdef CONFIG_SMP
		this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
		this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
		cpumask_set_cpu(cpu, mm_cpumask(next));

		/* Re-load page tables */
		load_cr3(next->pgd);
		trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);

		/* Stop flush ipis for the previous mm */
		cpumask_clear_cpu(cpu, mm_cpumask(prev));

		/* Load the LDT, if the LDT is different: */
		if (unlikely(prev->context.ldt != next->context.ldt))
			load_LDT_nolock(&next->context);
	}
#ifdef CONFIG_SMP
	  else {
		this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
		BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);

		if (!cpumask_test_cpu(cpu, mm_cpumask(next))) {
			/*
			 * On established mms, the mm_cpumask is only changed
			 * from irq context, from ptep_clear_flush() while in
			 * lazy tlb mode, and here. Irqs are blocked during
			 * schedule, protecting us from simultaneous changes.
			 */
			cpumask_set_cpu(cpu, mm_cpumask(next));
			/*
			 * We were in lazy tlb mode and leave_mm disabled
			 * tlb flush IPI delivery. We must reload CR3
			 * to make sure to use no freed page tables.
			 */
			load_cr3(next->pgd);
			trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
			load_LDT_nolock(&next->context);
		}
	}
#endif
}

#define activate_mm(prev, next)			\
do {						\
	paravirt_activate_mm((prev), (next));	\
	switch_mm((prev), (next), NULL);	\
} while (0);

#ifdef CONFIG_X86_32
#define deactivate_mm(tsk, mm)			\
do {						\
	lazy_load_gs(0);			\
} while (0)
#else
#define deactivate_mm(tsk, mm)			\
do {						\
	load_gs_index(0);			\
	loadsegment(fs, 0);			\
} while (0)
#endif

#endif /* _ASM_X86_MMU_CONTEXT_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_MMU_CONTEXT_H
1965aae3 PA	2	#define _ASM_X86_MMU_CONTEXT_H
c3c2fee3 JF	3
c3c2fee3 JF	4	#include <asm/desc.h>
60063497	5	#include <linux/atomic.h>
d17d8f9d DH	6	#include <linux/mm_types.h>
	7
	8	#include <trace/events/tlb.h>
	9
c3c2fee3 JF	10	#include <asm/pgalloc.h>
	11	#include <asm/tlbflush.h>
	12	#include <asm/paravirt.h>
	13	#ifndef CONFIG_PARAVIRT
	14	#include <asm-generic/mm_hooks.h>
	15
	16	static inline void paravirt_activate_mm(struct mm_struct *prev,
	17	struct mm_struct *next)
	18	{
	19	}
	20	#endif /* !CONFIG_PARAVIRT */
	21
	22	/*
	23	* Used for LDT copy/destruction.
	24	*/
	25	int init_new_context(struct task_struct tsk, struct mm_struct mm);
	26	void destroy_context(struct mm_struct *mm);
	27
6826c8ff BG	28
	29	static inline void enter_lazy_tlb(struct mm_struct mm, struct task_struct tsk)
	30	{
	31	#ifdef CONFIG_SMP
c6ae41e7 AS	32	if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
c6ae41e7 AS	33	this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
6826c8ff BG	34	#endif
	35	}
	36
	37	static inline void switch_mm(struct mm_struct prev, struct mm_struct next,
	38	struct task_struct *tsk)
	39	{
	40	unsigned cpu = smp_processor_id();
	41
	42	if (likely(prev != next)) {
6826c8ff	43	#ifdef CONFIG_SMP
c6ae41e7 AS	44	this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
c6ae41e7 AS	45	this_cpu_write(cpu_tlbstate.active_mm, next);
96a388de	46	#endif
78f1c4d6	47	cpumask_set_cpu(cpu, mm_cpumask(next));
6826c8ff BG	48
	49	/* Re-load page tables */
	50	load_cr3(next->pgd);
d17d8f9d	51	trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
6826c8ff	52
8f898fbb	53	/* Stop flush ipis for the previous mm */
831d52bc SS	54	cpumask_clear_cpu(cpu, mm_cpumask(prev));
831d52bc SS	55
8f898fbb	56	/* Load the LDT, if the LDT is different: */
6826c8ff BG	57	if (unlikely(prev->context.ldt != next->context.ldt))
	58	load_LDT_nolock(&next->context);
	59	}
	60	#ifdef CONFIG_SMP
8f898fbb	61	else {
c6ae41e7 AS	62	this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
c6ae41e7 AS	63	BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
6826c8ff	64
8f898fbb RR	65	if (!cpumask_test_cpu(cpu, mm_cpumask(next))) {
	66	/*
	67	* On established mms, the mm_cpumask is only changed
	68	* from irq context, from ptep_clear_flush() while in
	69	* lazy tlb mode, and here. Irqs are blocked during
	70	* schedule, protecting us from simultaneous changes.
	71	*/
	72	cpumask_set_cpu(cpu, mm_cpumask(next));
	73	/*
	74	* We were in lazy tlb mode and leave_mm disabled
6826c8ff BG	75	* tlb flush IPI delivery. We must reload CR3
	76	* to make sure to use no freed page tables.
	77	*/
	78	load_cr3(next->pgd);
d17d8f9d	79	trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
6826c8ff BG	80	load_LDT_nolock(&next->context);
	81	}
	82	}
	83	#endif
	84	}
c3c2fee3 JF	85
	86	#define activate_mm(prev, next) \
	87	do { \
	88	paravirt_activate_mm((prev), (next)); \
	89	switch_mm((prev), (next), NULL); \
	90	} while (0);
	91
6826c8ff BG	92	#ifdef CONFIG_X86_32
	93	#define deactivate_mm(tsk, mm) \
	94	do { \
ccbeed3a	95	lazy_load_gs(0); \
6826c8ff BG	96	} while (0)
	97	#else
	98	#define deactivate_mm(tsk, mm) \
	99	do { \
	100	load_gs_index(0); \
	101	loadsegment(fs, 0); \
	102	} while (0)
	103	#endif
c3c2fee3	104
1965aae3	105	#endif /* _ASM_X86_MMU_CONTEXT_H */