[deliverable/linux.git] / arch / arm64 / include / asm / cmpxchg.h

/*
 * Based on arch/arm/include/asm/cmpxchg.h
 *
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_CMPXCHG_H
#define __ASM_CMPXCHG_H

#include <linux/bug.h>
#include <linux/mmdebug.h>

#include <asm/atomic.h>
#include <asm/barrier.h>
#include <asm/lse.h>

/*
 * We need separate acquire parameters for ll/sc and lse, since the full
 * barrier case is generated as release+dmb for the former and
 * acquire+release for the latter.
 */
#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl)	\
static inline unsigned long __xchg_case_##name(unsigned long x,		\
					       volatile void *ptr)	\
{									\
	unsigned long ret, tmp;						\
									\
	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
	/* LL/SC */							\
	"	prfm	pstl1strm, %2\n"				\
	"1:	ld" #acq "xr" #sz "\t%" #w "0, %2\n"			\
	"	st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n"		\
	"	cbnz	%w1, 1b\n"					\
	"	" #mb,							\
	/* LSE atomics */						\
	"	nop\n"							\
	"	nop\n"							\
	"	swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n"	\
	"	nop\n"							\
	"	" #nop_lse)						\
	: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)			\
	: "r" (x)							\
	: cl);								\
									\
	return ret;							\
}

__XCHG_CASE(w, b,     1,        ,    ,  ,  ,  ,         )
__XCHG_CASE(w, h,     2,        ,    ,  ,  ,  ,         )
__XCHG_CASE(w,  ,     4,        ,    ,  ,  ,  ,         )
__XCHG_CASE( ,  ,     8,        ,    ,  ,  ,  ,         )
__XCHG_CASE(w, b, acq_1,        ,    , a, a,  , "memory")
__XCHG_CASE(w, h, acq_2,        ,    , a, a,  , "memory")
__XCHG_CASE(w,  , acq_4,        ,    , a, a,  , "memory")
__XCHG_CASE( ,  , acq_8,        ,    , a, a,  , "memory")
__XCHG_CASE(w, b, rel_1,        ,    ,  ,  , l, "memory")
__XCHG_CASE(w, h, rel_2,        ,    ,  ,  , l, "memory")
__XCHG_CASE(w,  , rel_4,        ,    ,  ,  , l, "memory")
__XCHG_CASE( ,  , rel_8,        ,    ,  ,  , l, "memory")
__XCHG_CASE(w, b,  mb_1, dmb ish, nop,  , a, l, "memory")
__XCHG_CASE(w, h,  mb_2, dmb ish, nop,  , a, l, "memory")
__XCHG_CASE(w,  ,  mb_4, dmb ish, nop,  , a, l, "memory")
__XCHG_CASE( ,  ,  mb_8, dmb ish, nop,  , a, l, "memory")

#undef __XCHG_CASE

#define __XCHG_GEN(sfx)							\
static inline unsigned long __xchg##sfx(unsigned long x,		\
					volatile void *ptr,		\
					int size)			\
{									\
	switch (size) {							\
	case 1:								\
		return __xchg_case##sfx##_1(x, ptr);			\
	case 2:								\
		return __xchg_case##sfx##_2(x, ptr);			\
	case 4:								\
		return __xchg_case##sfx##_4(x, ptr);			\
	case 8:								\
		return __xchg_case##sfx##_8(x, ptr);			\
	default:							\
		BUILD_BUG();						\
	}								\
									\
	unreachable();							\
}

__XCHG_GEN()
__XCHG_GEN(_acq)
__XCHG_GEN(_rel)
__XCHG_GEN(_mb)

#undef __XCHG_GEN

#define __xchg_wrapper(sfx, ptr, x)					\
({									\
	__typeof__(*(ptr)) __ret;					\
	__ret = (__typeof__(*(ptr)))					\
		__xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
	__ret;								\
})

/* xchg */
#define xchg_relaxed(...)	__xchg_wrapper(    , __VA_ARGS__)
#define xchg_acquire(...)	__xchg_wrapper(_acq, __VA_ARGS__)
#define xchg_release(...)	__xchg_wrapper(_rel, __VA_ARGS__)
#define xchg(...)		__xchg_wrapper( _mb, __VA_ARGS__)

#define __CMPXCHG_GEN(sfx)						\
static inline unsigned long __cmpxchg##sfx(volatile void *ptr,		\
					   unsigned long old,		\
					   unsigned long new,		\
					   int size)			\
{									\
	switch (size) {							\
	case 1:								\
		return __cmpxchg_case##sfx##_1(ptr, (u8)old, new);	\
	case 2:								\
		return __cmpxchg_case##sfx##_2(ptr, (u16)old, new);	\
	case 4:								\
		return __cmpxchg_case##sfx##_4(ptr, old, new);		\
	case 8:								\
		return __cmpxchg_case##sfx##_8(ptr, old, new);		\
	default:							\
		BUILD_BUG();						\
	}								\
									\
	unreachable();							\
}

__CMPXCHG_GEN()
__CMPXCHG_GEN(_acq)
__CMPXCHG_GEN(_rel)
__CMPXCHG_GEN(_mb)

#undef __CMPXCHG_GEN

#define __cmpxchg_wrapper(sfx, ptr, o, n)				\
({									\
	__typeof__(*(ptr)) __ret;					\
	__ret = (__typeof__(*(ptr)))					\
		__cmpxchg##sfx((ptr), (unsigned long)(o),		\
				(unsigned long)(n), sizeof(*(ptr)));	\
	__ret;								\
})

/* cmpxchg */
#define cmpxchg_relaxed(...)	__cmpxchg_wrapper(    , __VA_ARGS__)
#define cmpxchg_acquire(...)	__cmpxchg_wrapper(_acq, __VA_ARGS__)
#define cmpxchg_release(...)	__cmpxchg_wrapper(_rel, __VA_ARGS__)
#define cmpxchg(...)		__cmpxchg_wrapper( _mb, __VA_ARGS__)
#define cmpxchg_local		cmpxchg_relaxed

/* cmpxchg64 */
#define cmpxchg64_relaxed	cmpxchg_relaxed
#define cmpxchg64_acquire	cmpxchg_acquire
#define cmpxchg64_release	cmpxchg_release
#define cmpxchg64		cmpxchg
#define cmpxchg64_local		cmpxchg_local

/* cmpxchg_double */
#define system_has_cmpxchg_double()     1

#define __cmpxchg_double_check(ptr1, ptr2)					\
({										\
	if (sizeof(*(ptr1)) != 8)						\
		BUILD_BUG();							\
	VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1);	\
})

#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
({\
	int __ret;\
	__cmpxchg_double_check(ptr1, ptr2); \
	__ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
				     (unsigned long)(n1), (unsigned long)(n2), \
				     ptr1); \
	__ret; \
})

#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
({\
	int __ret;\
	__cmpxchg_double_check(ptr1, ptr2); \
	__ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
				  (unsigned long)(n1), (unsigned long)(n2), \
				  ptr1); \
	__ret; \
})

/* this_cpu_cmpxchg */
#define _protect_cmpxchg_local(pcp, o, n)			\
({								\
	typeof(*raw_cpu_ptr(&(pcp))) __ret;			\
	preempt_disable();					\
	__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);	\
	preempt_enable();					\
	__ret;							\
})

#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)

#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2)		\
({									\
	int __ret;							\
	preempt_disable();						\
	__ret = cmpxchg_double_local(	raw_cpu_ptr(&(ptr1)),		\
					raw_cpu_ptr(&(ptr2)),		\
					o1, o2, n1, n2);		\
	preempt_enable();						\
	__ret;								\
})

#endif	/* __ASM_CMPXCHG_H */
Commit	Line	Data
10b663ae CM	1	/*
	2	* Based on arch/arm/include/asm/cmpxchg.h
	3	*
	4	* Copyright (C) 2012 ARM Ltd.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18	#ifndef __ASM_CMPXCHG_H
	19	#define __ASM_CMPXCHG_H
	20
	21	#include <linux/bug.h>
5284e1b4	22	#include <linux/mmdebug.h>
10b663ae	23
c342f782	24	#include <asm/atomic.h>
10b663ae	25	#include <asm/barrier.h>
c8366ba0	26	#include <asm/lse.h>
10b663ae	27
305d454a WD	28	/*
	29	* We need separate acquire parameters for ll/sc and lse, since the full
	30	* barrier case is generated as release+dmb for the former and
	31	* acquire+release for the latter.
	32	*/
	33	#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \
	34	static inline unsigned long __xchg_case_##name(unsigned long x, \
	35	volatile void *ptr) \
	36	{ \
	37	unsigned long ret, tmp; \
	38	\
	39	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	40	/* LL/SC */ \
	41	" prfm pstl1strm, %2\n" \
	42	"1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \
	43	" st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \
	44	" cbnz %w1, 1b\n" \
	45	" " #mb, \
	46	/* LSE atomics */ \
	47	" nop\n" \
	48	" nop\n" \
	49	" swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \
	50	" nop\n" \
	51	" " #nop_lse) \
	52	: "=&r" (ret), "=&r" (tmp), "+Q" ((u8 )ptr) \
	53	: "r" (x) \
	54	: cl); \
	55	\
	56	return ret; \
10b663ae CM	57	}
10b663ae CM	58
305d454a WD	59	__XCHG_CASE(w, b, 1, , , , , , )
	60	__XCHG_CASE(w, h, 2, , , , , , )
	61	__XCHG_CASE(w, , 4, , , , , , )
	62	__XCHG_CASE( , , 8, , , , , , )
	63	__XCHG_CASE(w, b, acq_1, , , a, a, , "memory")
	64	__XCHG_CASE(w, h, acq_2, , , a, a, , "memory")
	65	__XCHG_CASE(w, , acq_4, , , a, a, , "memory")
	66	__XCHG_CASE( , , acq_8, , , a, a, , "memory")
	67	__XCHG_CASE(w, b, rel_1, , , , , l, "memory")
	68	__XCHG_CASE(w, h, rel_2, , , , , l, "memory")
	69	__XCHG_CASE(w, , rel_4, , , , , l, "memory")
	70	__XCHG_CASE( , , rel_8, , , , , l, "memory")
	71	__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory")
	72	__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory")
	73	__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory")
	74	__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory")
	75
	76	#undef __XCHG_CASE
	77
	78	#define __XCHG_GEN(sfx) \
	79	static inline unsigned long __xchg##sfx(unsigned long x, \
	80	volatile void *ptr, \
	81	int size) \
	82	{ \
	83	switch (size) { \
	84	case 1: \
	85	return __xchg_case##sfx##_1(x, ptr); \
	86	case 2: \
	87	return __xchg_case##sfx##_2(x, ptr); \
	88	case 4: \
	89	return __xchg_case##sfx##_4(x, ptr); \
	90	case 8: \
	91	return __xchg_case##sfx##_8(x, ptr); \
	92	default: \
	93	BUILD_BUG(); \
	94	} \
	95	\
	96	unreachable(); \
	97	}
	98
	99	__XCHG_GEN()
	100	__XCHG_GEN(_acq)
	101	__XCHG_GEN(_rel)
	102	__XCHG_GEN(_mb)
	103
	104	#undef __XCHG_GEN
	105
	106	#define __xchg_wrapper(sfx, ptr, x) \
	107	({ \
	108	__typeof__(*(ptr)) __ret; \
	109	__ret = (__typeof__(*(ptr))) \
	110	__xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
	111	__ret; \
e1dfda9c	112	})
10b663ae	113
305d454a WD	114	/* xchg */
	115	#define xchg_relaxed(...) __xchg_wrapper( , __VA_ARGS__)
	116	#define xchg_acquire(...) __xchg_wrapper(_acq, __VA_ARGS__)
	117	#define xchg_release(...) __xchg_wrapper(_rel, __VA_ARGS__)
	118	#define xchg(...) __xchg_wrapper( _mb, __VA_ARGS__)
	119
	120	#define __CMPXCHG_GEN(sfx) \
	121	static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
	122	unsigned long old, \
	123	unsigned long new, \
	124	int size) \
	125	{ \
	126	switch (size) { \
	127	case 1: \
	128	return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \
	129	case 2: \
	130	return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \
	131	case 4: \
	132	return __cmpxchg_case##sfx##_4(ptr, old, new); \
	133	case 8: \
	134	return __cmpxchg_case##sfx##_8(ptr, old, new); \
	135	default: \
	136	BUILD_BUG(); \
	137	} \
	138	\
	139	unreachable(); \
10b663ae CM	140	}
10b663ae CM	141
305d454a WD	142	__CMPXCHG_GEN()
	143	__CMPXCHG_GEN(_acq)
	144	__CMPXCHG_GEN(_rel)
	145	__CMPXCHG_GEN(_mb)
10b663ae	146
305d454a	147	#undef __CMPXCHG_GEN
60010e50	148
305d454a WD	149	#define __cmpxchg_wrapper(sfx, ptr, o, n) \
	150	({ \
	151	__typeof__(*(ptr)) __ret; \
	152	__ret = (__typeof__(*(ptr))) \
	153	__cmpxchg##sfx((ptr), (unsigned long)(o), \
	154	(unsigned long)(n), sizeof(*(ptr))); \
	155	__ret; \
60010e50	156	})
10b663ae	157
305d454a WD	158	/* cmpxchg */
	159	#define cmpxchg_relaxed(...) __cmpxchg_wrapper( , __VA_ARGS__)
	160	#define cmpxchg_acquire(...) __cmpxchg_wrapper(_acq, __VA_ARGS__)
	161	#define cmpxchg_release(...) __cmpxchg_wrapper(_rel, __VA_ARGS__)
	162	#define cmpxchg(...) __cmpxchg_wrapper( _mb, __VA_ARGS__)
	163	#define cmpxchg_local cmpxchg_relaxed
	164
	165	/* cmpxchg64 */
	166	#define cmpxchg64_relaxed cmpxchg_relaxed
	167	#define cmpxchg64_acquire cmpxchg_acquire
	168	#define cmpxchg64_release cmpxchg_release
	169	#define cmpxchg64 cmpxchg
	170	#define cmpxchg64_local cmpxchg_local
	171
	172	/* cmpxchg_double */
e9a4b795 WD	173	#define system_has_cmpxchg_double() 1
	174
	175	#define __cmpxchg_double_check(ptr1, ptr2) \
	176	({ \
	177	if (sizeof(*(ptr1)) != 8) \
	178	BUILD_BUG(); \
	179	VM_BUG_ON((unsigned long )(ptr2) - (unsigned long )(ptr1) != 1); \
	180	})
	181
5284e1b4 SC	182	#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
	183	({\
	184	int __ret;\
e9a4b795 WD	185	__cmpxchg_double_check(ptr1, ptr2); \
	186	__ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
	187	(unsigned long)(n1), (unsigned long)(n2), \
	188	ptr1); \
5284e1b4 SC	189	__ret; \
	190	})
	191
	192	#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
	193	({\
	194	int __ret;\
e9a4b795 WD	195	__cmpxchg_double_check(ptr1, ptr2); \
	196	__ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
	197	(unsigned long)(n1), (unsigned long)(n2), \
	198	ptr1); \
5284e1b4 SC	199	__ret; \
	200	})
	201
305d454a	202	/* this_cpu_cmpxchg */
f3eab718 SC	203	#define _protect_cmpxchg_local(pcp, o, n) \
	204	({ \
	205	typeof(*raw_cpu_ptr(&(pcp))) __ret; \
	206	preempt_disable(); \
	207	__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
	208	preempt_enable(); \
	209	__ret; \
	210	})
	211
	212	#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
	213	#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
	214	#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
	215	#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
	216
	217	#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
	218	({ \
	219	int __ret; \
	220	preempt_disable(); \
	221	__ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
	222	raw_cpu_ptr(&(ptr2)), \
	223	o1, o2, n1, n2); \
	224	preempt_enable(); \
	225	__ret; \
	226	})
5284e1b4	227
10b663ae	228	#endif /* __ASM_CMPXCHG_H */