[deliverable/linux.git] / arch / x86 / include / asm / percpu.h

#ifndef _ASM_X86_PERCPU_H
#define _ASM_X86_PERCPU_H

#ifdef CONFIG_X86_64
#define __percpu_seg		gs
#define __percpu_mov_op		movq
#else
#define __percpu_seg		fs
#define __percpu_mov_op		movl
#endif

#ifdef __ASSEMBLY__

/*
 * PER_CPU finds an address of a per-cpu variable.
 *
 * Args:
 *    var - variable name
 *    reg - 32bit register
 *
 * The resulting address is stored in the "reg" argument.
 *
 * Example:
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
#ifdef CONFIG_SMP
#define PER_CPU(var, reg)						\
	__percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg;	\
	lea per_cpu__##var(reg), reg
#define PER_CPU_VAR(var)	%__percpu_seg:per_cpu__##var
#else /* ! SMP */
#define PER_CPU(var, reg)						\
	__percpu_mov_op $per_cpu__##var, reg
#define PER_CPU_VAR(var)	per_cpu__##var
#endif	/* SMP */

#else /* ...!ASSEMBLY */

#include <linux/stringify.h>

#ifdef CONFIG_SMP
#define __percpu_arg(x)		"%%"__stringify(__percpu_seg)":%P" #x
#define __my_cpu_offset		percpu_read(this_cpu_off)
#else
#define __percpu_arg(x)		"%" #x
#endif

/* For arch-specific code, we can use direct single-insn ops (they
 * don't give an lvalue though). */
extern void __bad_percpu_size(void);

#define percpu_to_op(op, var, val)			\
do {							\
	typedef typeof(var) T__;			\
	if (0) {					\
		T__ tmp__;				\
		tmp__ = (val);				\
	}						\
	switch (sizeof(var)) {				\
	case 1:						\
		asm(op "b %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "ri" ((T__)val));			\
		break;					\
	case 2:						\
		asm(op "w %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "ri" ((T__)val));			\
		break;					\
	case 4:						\
		asm(op "l %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "ri" ((T__)val));			\
		break;					\
	case 8:						\
		asm(op "q %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "re" ((T__)val));			\
		break;					\
	default: __bad_percpu_size();			\
	}						\
} while (0)

#define percpu_from_op(op, var)				\
({							\
	typeof(var) ret__;				\
	switch (sizeof(var)) {				\
	case 1:						\
		asm(op "b "__percpu_arg(1)",%0"		\
		    : "=r" (ret__)			\
		    : "m" (var));			\
		break;					\
	case 2:						\
		asm(op "w "__percpu_arg(1)",%0"		\
		    : "=r" (ret__)			\
		    : "m" (var));			\
		break;					\
	case 4:						\
		asm(op "l "__percpu_arg(1)",%0"		\
		    : "=r" (ret__)			\
		    : "m" (var));			\
		break;					\
	case 8:						\
		asm(op "q "__percpu_arg(1)",%0"		\
		    : "=r" (ret__)			\
		    : "m" (var));			\
		break;					\
	default: __bad_percpu_size();			\
	}						\
	ret__;						\
})

#define percpu_read(var)	percpu_from_op("mov", per_cpu__##var)
#define percpu_write(var, val)	percpu_to_op("mov", per_cpu__##var, val)
#define percpu_add(var, val)	percpu_to_op("add", per_cpu__##var, val)
#define percpu_sub(var, val)	percpu_to_op("sub", per_cpu__##var, val)
#define percpu_and(var, val)	percpu_to_op("and", per_cpu__##var, val)
#define percpu_or(var, val)	percpu_to_op("or", per_cpu__##var, val)
#define percpu_xor(var, val)	percpu_to_op("xor", per_cpu__##var, val)

/* This is not atomic against other CPUs -- CPU preemption needs to be off */
#define x86_test_and_clear_bit_percpu(bit, var)				\
({									\
	int old__;							\
	asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0"		\
		     : "=r" (old__), "+m" (per_cpu__##var)		\
		     : "dIr" (bit));					\
	old__;								\
})

#include <asm-generic/percpu.h>

/* We can use this directly for local CPU (faster). */
DECLARE_PER_CPU(unsigned long, this_cpu_off);

#endif /* !__ASSEMBLY__ */

#ifdef CONFIG_SMP

/*
 * Define the "EARLY_PER_CPU" macros.  These are used for some per_cpu
 * variables that are initialized and accessed before there are per_cpu
 * areas allocated.
 */

#define	DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)			\
	DEFINE_PER_CPU(_type, _name) = _initvalue;			\
	__typeof__(_type) _name##_early_map[NR_CPUS] __initdata =	\
				{ [0 ... NR_CPUS-1] = _initvalue };	\
	__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map

#define EXPORT_EARLY_PER_CPU_SYMBOL(_name)			\
	EXPORT_PER_CPU_SYMBOL(_name)

#define DECLARE_EARLY_PER_CPU(_type, _name)			\
	DECLARE_PER_CPU(_type, _name);				\
	extern __typeof__(_type) *_name##_early_ptr;		\
	extern __typeof__(_type)  _name##_early_map[]

#define	early_per_cpu_ptr(_name) (_name##_early_ptr)
#define	early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
#define	early_per_cpu(_name, _cpu) 				\
	*(early_per_cpu_ptr(_name) ?				\
		&early_per_cpu_ptr(_name)[_cpu] :		\
		&per_cpu(_name, _cpu))

#else	/* !CONFIG_SMP */
#define	DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)		\
	DEFINE_PER_CPU(_type, _name) = _initvalue

#define EXPORT_EARLY_PER_CPU_SYMBOL(_name)			\
	EXPORT_PER_CPU_SYMBOL(_name)

#define DECLARE_EARLY_PER_CPU(_type, _name)			\
	DECLARE_PER_CPU(_type, _name)

#define	early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
#define	early_per_cpu_ptr(_name) NULL
/* no early_per_cpu_map() */

#endif	/* !CONFIG_SMP */

#endif /* _ASM_X86_PERCPU_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_PERCPU_H
1965aae3 PA	2	#define _ASM_X86_PERCPU_H
3334052a	3
1a51e3a0	4	#ifdef CONFIG_X86_64
9939ddaf TH	5	#define __percpu_seg gs
9939ddaf TH	6	#define __percpu_mov_op movq
1a51e3a0	7	#else
9939ddaf TH	8	#define __percpu_seg fs
9939ddaf TH	9	#define __percpu_mov_op movl
96a388de	10	#endif
3334052a	11
	12	#ifdef __ASSEMBLY__
	13
	14	/*
	15	* PER_CPU finds an address of a per-cpu variable.
	16	*
	17	* Args:
	18	* var - variable name
	19	* reg - 32bit register
	20	*
	21	* The resulting address is stored in the "reg" argument.
	22	*
	23	* Example:
	24	* PER_CPU(cpu_gdt_descr, %ebx)
	25	*/
	26	#ifdef CONFIG_SMP
9939ddaf TH	27	#define PER_CPU(var, reg) \
9939ddaf TH	28	__percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \
3334052a	29	lea per_cpu__##var(reg), reg
9939ddaf	30	#define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var
3334052a	31	#else /* ! SMP */
9939ddaf TH	32	#define PER_CPU(var, reg) \
9939ddaf TH	33	__percpu_mov_op $per_cpu__##var, reg
3334052a	34	#define PER_CPU_VAR(var) per_cpu__##var
	35	#endif /* SMP */
	36
	37	#else /* ...!ASSEMBLY */
	38
9939ddaf	39	#include <linux/stringify.h>
3334052a	40
9939ddaf	41	#ifdef CONFIG_SMP
87b26406	42	#define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x
6dbde353	43	#define __my_cpu_offset percpu_read(this_cpu_off)
9939ddaf	44	#else
87b26406	45	#define __percpu_arg(x) "%" #x
9939ddaf	46	#endif
3334052a	47
3334052a	48	/* For arch-specific code, we can use direct single-insn ops (they
	49	* don't give an lvalue though). */
	50	extern void __bad_percpu_size(void);
	51
bc9e3be2 JP	52	#define percpu_to_op(op, var, val) \
	53	do { \
	54	typedef typeof(var) T__; \
	55	if (0) { \
	56	T__ tmp__; \
	57	tmp__ = (val); \
	58	} \
	59	switch (sizeof(var)) { \
	60	case 1: \
87b26406	61	asm(op "b %1,"__percpu_arg(0) \
bc9e3be2 JP	62	: "+m" (var) \
	63	: "ri" ((T__)val)); \
	64	break; \
	65	case 2: \
87b26406	66	asm(op "w %1,"__percpu_arg(0) \
bc9e3be2 JP	67	: "+m" (var) \
	68	: "ri" ((T__)val)); \
	69	break; \
	70	case 4: \
87b26406	71	asm(op "l %1,"__percpu_arg(0) \
bc9e3be2 JP	72	: "+m" (var) \
	73	: "ri" ((T__)val)); \
	74	break; \
9939ddaf	75	case 8: \
87b26406	76	asm(op "q %1,"__percpu_arg(0) \
9939ddaf	77	: "+m" (var) \
299e2699	78	: "re" ((T__)val)); \
9939ddaf	79	break; \
bc9e3be2 JP	80	default: __bad_percpu_size(); \
	81	} \
	82	} while (0)
	83
	84	#define percpu_from_op(op, var) \
	85	({ \
	86	typeof(var) ret__; \
	87	switch (sizeof(var)) { \
	88	case 1: \
87b26406	89	asm(op "b "__percpu_arg(1)",%0" \
bc9e3be2 JP	90	: "=r" (ret__) \
	91	: "m" (var)); \
	92	break; \
	93	case 2: \
87b26406	94	asm(op "w "__percpu_arg(1)",%0" \
bc9e3be2 JP	95	: "=r" (ret__) \
	96	: "m" (var)); \
	97	break; \
	98	case 4: \
87b26406	99	asm(op "l "__percpu_arg(1)",%0" \
9939ddaf TH	100	: "=r" (ret__) \
	101	: "m" (var)); \
	102	break; \
	103	case 8: \
87b26406	104	asm(op "q "__percpu_arg(1)",%0" \
bc9e3be2 JP	105	: "=r" (ret__) \
	106	: "m" (var)); \
	107	break; \
	108	default: __bad_percpu_size(); \
	109	} \
	110	ret__; \
	111	})
3334052a	112
6dbde353 IM	113	#define percpu_read(var) percpu_from_op("mov", per_cpu__##var)
	114	#define percpu_write(var, val) percpu_to_op("mov", per_cpu__##var, val)
	115	#define percpu_add(var, val) percpu_to_op("add", per_cpu__##var, val)
	116	#define percpu_sub(var, val) percpu_to_op("sub", per_cpu__##var, val)
	117	#define percpu_and(var, val) percpu_to_op("and", per_cpu__##var, val)
	118	#define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val)
	119	#define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val)
9939ddaf	120
49357d19 TH	121	/* This is not atomic against other CPUs -- CPU preemption needs to be off */
	122	#define x86_test_and_clear_bit_percpu(bit, var) \
	123	({ \
	124	int old__; \
87b26406 BG	125	asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \
	126	: "=r" (old__), "+m" (per_cpu__##var) \
	127	: "dIr" (bit)); \
49357d19 TH	128	old__; \
	129	})
	130
6dbde353 IM	131	#include <asm-generic/percpu.h>
	132
	133	/* We can use this directly for local CPU (faster). */
	134	DECLARE_PER_CPU(unsigned long, this_cpu_off);
	135
3334052a	136	#endif /* !__ASSEMBLY__ */
23ca4bba MT	137
	138	#ifdef CONFIG_SMP
	139
	140	/*
	141	* Define the "EARLY_PER_CPU" macros. These are used for some per_cpu
	142	* variables that are initialized and accessed before there are per_cpu
	143	* areas allocated.
	144	*/
	145
	146	#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
	147	DEFINE_PER_CPU(_type, _name) = _initvalue; \
	148	__typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
	149	{ [0 ... NR_CPUS-1] = _initvalue }; \
c6a92a25	150	__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
23ca4bba MT	151
	152	#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
	153	EXPORT_PER_CPU_SYMBOL(_name)
	154
	155	#define DECLARE_EARLY_PER_CPU(_type, _name) \
	156	DECLARE_PER_CPU(_type, _name); \
	157	extern __typeof__(_type) *_name##_early_ptr; \
	158	extern __typeof__(_type) _name##_early_map[]
	159
	160	#define early_per_cpu_ptr(_name) (_name##_early_ptr)
	161	#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
	162	#define early_per_cpu(_name, _cpu) \
f10fcd47 TH	163	*(early_per_cpu_ptr(_name) ? \
	164	&early_per_cpu_ptr(_name)[_cpu] : \
	165	&per_cpu(_name, _cpu))
23ca4bba MT	166
	167	#else /* !CONFIG_SMP */
	168	#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
	169	DEFINE_PER_CPU(_type, _name) = _initvalue
	170
	171	#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
	172	EXPORT_PER_CPU_SYMBOL(_name)
	173
	174	#define DECLARE_EARLY_PER_CPU(_type, _name) \
	175	DECLARE_PER_CPU(_type, _name)
	176
	177	#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
	178	#define early_per_cpu_ptr(_name) NULL
	179	/* no early_per_cpu_map() */
	180
	181	#endif /* !CONFIG_SMP */
	182
1965aae3	183	#endif /* _ASM_X86_PERCPU_H */