[deliverable/linux.git] / arch / s390 / include / asm / percpu.h

#ifndef __ARCH_S390_PERCPU__
#define __ARCH_S390_PERCPU__

#include <linux/preempt.h>
#include <asm/cmpxchg.h>

/*
 * s390 uses its own implementation for per cpu data, the offset of
 * the cpu local data area is cached in the cpu's lowcore memory.
 */
#define __my_cpu_offset S390_lowcore.percpu_offset

#ifdef CONFIG_64BIT

/*
 * For 64 bit module code, the module may be more than 4G above the
 * per cpu area, use weak definitions to force the compiler to
 * generate external references.
 */
#if defined(CONFIG_SMP) && defined(MODULE)
#define ARCH_NEEDS_WEAK_PER_CPU
#endif

/*
 * We use a compare-and-swap loop since that uses less cpu cycles than
 * disabling and enabling interrupts like the generic variant would do.
 */
#define arch_this_cpu_to_op_simple(pcp, val, op)			\
({									\
	typedef typeof(pcp) pcp_op_T__;					\
	pcp_op_T__ old__, new__, prev__;				\
	pcp_op_T__ *ptr__;						\
	preempt_disable();						\
	ptr__ = __this_cpu_ptr(&(pcp));					\
	prev__ = *ptr__;						\
	do {								\
		old__ = prev__;						\
		new__ = old__ op (val);					\
		prev__ = cmpxchg(ptr__, old__, new__);			\
	} while (prev__ != old__);					\
	preempt_enable();						\
	new__;								\
})

#define this_cpu_add_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_and_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
#define this_cpu_and_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
#define this_cpu_or_1(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
#define this_cpu_or_2(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)

#ifndef CONFIG_HAVE_MARCH_Z196_FEATURES

#define this_cpu_add_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
#define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
#define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
#define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
#define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)

#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */

#define arch_this_cpu_add(pcp, val, op1, op2, szcast)			\
{									\
	typedef typeof(pcp) pcp_op_T__; 				\
	pcp_op_T__ val__ = (val);					\
	pcp_op_T__ old__, *ptr__;					\
	preempt_disable();						\
	ptr__ = __this_cpu_ptr(&(pcp)); 				\
	if (__builtin_constant_p(val__) &&				\
	    ((szcast)val__ > -129) && ((szcast)val__ < 128)) {		\
		asm volatile(						\
			op2 "   %[ptr__],%[val__]\n"			\
			: [ptr__] "+Q" (*ptr__) 			\
			: [val__] "i" ((szcast)val__)			\
			: "cc");					\
	} else {							\
		asm volatile(						\
			op1 "   %[old__],%[val__],%[ptr__]\n"		\
			: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)	\
			: [val__] "d" (val__)				\
			: "cc");					\
	}								\
	preempt_enable();						\
}

#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)

#define arch_this_cpu_add_return(pcp, val, op)				\
({									\
	typedef typeof(pcp) pcp_op_T__; 				\
	pcp_op_T__ val__ = (val);					\
	pcp_op_T__ old__, *ptr__;					\
	preempt_disable();						\
	ptr__ = __this_cpu_ptr(&(pcp)); 				\
	asm volatile(							\
		op "    %[old__],%[val__],%[ptr__]\n"			\
		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
		: [val__] "d" (val__)					\
		: "cc");						\
	preempt_enable();						\
	old__ + val__;							\
})

#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")

#define arch_this_cpu_to_op(pcp, val, op)				\
{									\
	typedef typeof(pcp) pcp_op_T__; 				\
	pcp_op_T__ val__ = (val);					\
	pcp_op_T__ old__, *ptr__;					\
	preempt_disable();						\
	ptr__ = __this_cpu_ptr(&(pcp)); 				\
	asm volatile(							\
		op "    %[old__],%[val__],%[ptr__]\n"			\
		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
		: [val__] "d" (val__)					\
		: "cc");						\
	preempt_enable();						\
}

#define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op(pcp, val, "lan")
#define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op(pcp, val, "lang")
#define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op(pcp, val, "lao")
#define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op(pcp, val, "laog")

#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */

#define arch_this_cpu_cmpxchg(pcp, oval, nval)				\
({									\
	typedef typeof(pcp) pcp_op_T__;					\
	pcp_op_T__ ret__;						\
	pcp_op_T__ *ptr__;						\
	preempt_disable();						\
	ptr__ = __this_cpu_ptr(&(pcp));					\
	ret__ = cmpxchg(ptr__, oval, nval);				\
	preempt_enable();						\
	ret__;								\
})

#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)

#define arch_this_cpu_xchg(pcp, nval)					\
({									\
	typeof(pcp) *ptr__;						\
	typeof(pcp) ret__;						\
	preempt_disable();						\
	ptr__ = __this_cpu_ptr(&(pcp));					\
	ret__ = xchg(ptr__, nval);					\
	preempt_enable();						\
	ret__;								\
})

#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)

#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2)	\
({									\
	typeof(pcp1) o1__ = (o1), n1__ = (n1);				\
	typeof(pcp2) o2__ = (o2), n2__ = (n2);				\
	typeof(pcp1) *p1__;						\
	typeof(pcp2) *p2__;						\
	int ret__;							\
	preempt_disable();						\
	p1__ = __this_cpu_ptr(&(pcp1));					\
	p2__ = __this_cpu_ptr(&(pcp2));					\
	ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__);	\
	preempt_enable();						\
	ret__;								\
})

#define this_cpu_cmpxchg_double_4 arch_this_cpu_cmpxchg_double
#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double

#endif /* CONFIG_64BIT */

#include <asm-generic/percpu.h>

#endif /* __ARCH_S390_PERCPU__ */
Commit	Line	Data
1da177e4 LT	1	#ifndef __ARCH_S390_PERCPU__
	2	#define __ARCH_S390_PERCPU__
	3
4c2241fd HC	4	#include <linux/preempt.h>
	5	#include <asm/cmpxchg.h>
	6
1da177e4 LT	7	/*
	8	* s390 uses its own implementation for per cpu data, the offset of
	9	* the cpu local data area is cached in the cpu's lowcore memory.
1da177e4	10	*/
9a0ef292	11	#define __my_cpu_offset S390_lowcore.percpu_offset
1da177e4	12
0702fbf5 HC	13	#ifdef CONFIG_64BIT
0702fbf5 HC	14
9a0ef292 TH	15	/*
	16	* For 64 bit module code, the module may be more than 4G above the
	17	* per cpu area, use weak definitions to force the compiler to
	18	* generate external references.
	19	*/
0702fbf5	20	#if defined(CONFIG_SMP) && defined(MODULE)
9a0ef292	21	#define ARCH_NEEDS_WEAK_PER_CPU
1da177e4 LT	22	#endif
1da177e4 LT	23
f84cd97e HC	24	/*
	25	* We use a compare-and-swap loop since that uses less cpu cycles than
	26	* disabling and enabling interrupts like the generic variant would do.
	27	*/
	28	#define arch_this_cpu_to_op_simple(pcp, val, op) \
ba6f5c2a	29	({ \
4c2241fd HC	30	typedef typeof(pcp) pcp_op_T__; \
	31	pcp_op_T__ old__, new__, prev__; \
	32	pcp_op_T__ *ptr__; \
	33	preempt_disable(); \
	34	ptr__ = __this_cpu_ptr(&(pcp)); \
	35	prev__ = *ptr__; \
	36	do { \
	37	old__ = prev__; \
	38	new__ = old__ op (val); \
0702fbf5	39	prev__ = cmpxchg(ptr__, old__, new__); \
4c2241fd HC	40	} while (prev__ != old__); \
4c2241fd HC	41	preempt_enable(); \
ba6f5c2a HC	42	new__; \
ba6f5c2a HC	43	})
4c2241fd	44
f84cd97e HC	45	#define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	46	#define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	47	#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	48	#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	49	#define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
	50	#define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
	51	#define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, \|)
	52	#define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, \|)
f84cd97e HC	53
	54	#ifndef CONFIG_HAVE_MARCH_Z196_FEATURES
	55
	56	#define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	57	#define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	58	#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	59	#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
	60	#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
	61	#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
	62	#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, \|)
	63	#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, \|)
f84cd97e HC	64
	65	#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
	66
	67	#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \
	68	{ \
	69	typedef typeof(pcp) pcp_op_T__; \
	70	pcp_op_T__ val__ = (val); \
	71	pcp_op_T__ old__, *ptr__; \
	72	preempt_disable(); \
	73	ptr__ = __this_cpu_ptr(&(pcp)); \
	74	if (__builtin_constant_p(val__) && \
	75	((szcast)val__ > -129) && ((szcast)val__ < 128)) { \
	76	asm volatile( \
	77	op2 " %[ptr__],%[val__]\n" \
	78	: [ptr__] "+Q" (*ptr__) \
	79	: [val__] "i" ((szcast)val__) \
	80	: "cc"); \
	81	} else { \
	82	asm volatile( \
	83	op1 " %[old__],%[val__],%[ptr__]\n" \
	84	: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
	85	: [val__] "d" (val__) \
	86	: "cc"); \
	87	} \
	88	preempt_enable(); \
	89	}
	90
	91	#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
	92	#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
	93
	94	#define arch_this_cpu_add_return(pcp, val, op) \
	95	({ \
	96	typedef typeof(pcp) pcp_op_T__; \
	97	pcp_op_T__ val__ = (val); \
	98	pcp_op_T__ old__, *ptr__; \
	99	preempt_disable(); \
	100	ptr__ = __this_cpu_ptr(&(pcp)); \
	101	asm volatile( \
	102	op " %[old__],%[val__],%[ptr__]\n" \
	103	: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
	104	: [val__] "d" (val__) \
	105	: "cc"); \
	106	preempt_enable(); \
	107	old__ + val__; \
	108	})
	109
	110	#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
	111	#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
	112
	113	#define arch_this_cpu_to_op(pcp, val, op) \
	114	{ \
	115	typedef typeof(pcp) pcp_op_T__; \
	116	pcp_op_T__ val__ = (val); \
	117	pcp_op_T__ old__, *ptr__; \
	118	preempt_disable(); \
	119	ptr__ = __this_cpu_ptr(&(pcp)); \
	120	asm volatile( \
	121	op " %[old__],%[val__],%[ptr__]\n" \
	122	: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
	123	: [val__] "d" (val__) \
	124	: "cc"); \
	125	preempt_enable(); \
	126	}
	127
128	#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan")
129	#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang")
130	#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao")
131	#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog")
f84cd97e HC	132
f84cd97e HC	133	#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
4c2241fd	134
b1d6b40c	135	#define arch_this_cpu_cmpxchg(pcp, oval, nval) \
4c2241fd HC	136	({ \
	137	typedef typeof(pcp) pcp_op_T__; \
	138	pcp_op_T__ ret__; \
	139	pcp_op_T__ *ptr__; \
	140	preempt_disable(); \
	141	ptr__ = __this_cpu_ptr(&(pcp)); \
0702fbf5	142	ret__ = cmpxchg(ptr__, oval, nval); \
4c2241fd HC	143	preempt_enable(); \
	144	ret__; \
	145	})
	146
933393f5 CL	147	#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
	148	#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
	149	#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
	150	#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
4c2241fd	151
28634a07 HC	152	#define arch_this_cpu_xchg(pcp, nval) \
	153	({ \
	154	typeof(pcp) *ptr__; \
	155	typeof(pcp) ret__; \
	156	preempt_disable(); \
	157	ptr__ = __this_cpu_ptr(&(pcp)); \
	158	ret__ = xchg(ptr__, nval); \
	159	preempt_enable(); \
	160	ret__; \
	161	})
	162
	163	#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
	164	#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
	165	#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
28634a07	166	#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
28634a07	167
b1d6b40c HC	168	#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \
	169	({ \
	170	typeof(pcp1) o1__ = (o1), n1__ = (n1); \
	171	typeof(pcp2) o2__ = (o2), n2__ = (n2); \
	172	typeof(pcp1) *p1__; \
	173	typeof(pcp2) *p2__; \
	174	int ret__; \
	175	preempt_disable(); \
	176	p1__ = __this_cpu_ptr(&(pcp1)); \
	177	p2__ = __this_cpu_ptr(&(pcp2)); \
	178	ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__); \
	179	preempt_enable(); \
	180	ret__; \
	181	})
	182
	183	#define this_cpu_cmpxchg_double_4 arch_this_cpu_cmpxchg_double
b1d6b40c	184	#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
0702fbf5 HC	185
0702fbf5 HC	186	#endif /* CONFIG_64BIT */
b1d6b40c	187
f0343474	188	#include <asm-generic/percpu.h>
1da177e4	189
1da177e4	190	#endif /* __ARCH_S390_PERCPU__ */