[deliverable/linux.git] / arch / arm64 / include / asm / atomic_ll_sc.h

/*
 * Based on arch/arm/include/asm/atomic.h
 *
 * Copyright (C) 1996 Russell King.
 * Copyright (C) 2002 Deep Blue Solutions Ltd.
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef __ASM_ATOMIC_LL_SC_H
#define __ASM_ATOMIC_LL_SC_H

#ifndef __ARM64_IN_ATOMIC_IMPL
#error "please don't include this file directly"
#endif

/*
 * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
 * store exclusive to ensure that these are atomic.  We may loop
 * to ensure that the update happens.
 *
 * NOTE: these functions do *not* follow the PCS and must explicitly
 * save any clobbered registers other than x0 (regardless of return
 * value).  This is achieved through -fcall-saved-* compiler flags for
 * this file, which unfortunately don't work on a per-function basis
 * (the optimize attribute silently ignores these options).
 */

#define ATOMIC_OP(op, asm_op)						\
__LL_SC_INLINE void							\
__LL_SC_PREFIX(atomic_##op(int i, atomic_t *v))				\
{									\
	unsigned long tmp;						\
	int result;							\
									\
	asm volatile("// atomic_" #op "\n"				\
"1:	ldxr	%w0, %2\n"						\
"	" #asm_op "	%w0, %w0, %w3\n"				\
"	stxr	%w1, %w0, %2\n"						\
"	cbnz	%w1, 1b"						\
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
	: "Ir" (i));							\
}									\
__LL_SC_EXPORT(atomic_##op);

#define ATOMIC_OP_RETURN(op, asm_op)					\
__LL_SC_INLINE int							\
__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v))		\
{									\
	unsigned long tmp;						\
	int result;							\
									\
	asm volatile("// atomic_" #op "_return\n"			\
"1:	ldxr	%w0, %2\n"						\
"	" #asm_op "	%w0, %w0, %w3\n"				\
"	stlxr	%w1, %w0, %2\n"						\
"	cbnz	%w1, 1b"						\
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
	: "Ir" (i)							\
	: "memory");							\
									\
	smp_mb();							\
	return result;							\
}									\
__LL_SC_EXPORT(atomic_##op##_return);

#define ATOMIC_OPS(op, asm_op)						\
	ATOMIC_OP(op, asm_op)						\
	ATOMIC_OP_RETURN(op, asm_op)

ATOMIC_OPS(add, add)
ATOMIC_OPS(sub, sub)

ATOMIC_OP(and, and)
ATOMIC_OP(andnot, bic)
ATOMIC_OP(or, orr)
ATOMIC_OP(xor, eor)

#undef ATOMIC_OPS
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP

__LL_SC_INLINE int
__LL_SC_PREFIX(atomic_cmpxchg(atomic_t *ptr, int old, int new))
{
	unsigned long tmp;
	int oldval;

	smp_mb();

	asm volatile("// atomic_cmpxchg\n"
"1:	ldxr	%w1, %2\n"
"	cmp	%w1, %w3\n"
"	b.ne	2f\n"
"	stxr	%w0, %w4, %2\n"
"	cbnz	%w0, 1b\n"
"2:"
	: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
	: "Ir" (old), "r" (new)
	: "cc");

	smp_mb();
	return oldval;
}
__LL_SC_EXPORT(atomic_cmpxchg);

#define ATOMIC64_OP(op, asm_op)						\
__LL_SC_INLINE void							\
__LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v))			\
{									\
	long result;							\
	unsigned long tmp;						\
									\
	asm volatile("// atomic64_" #op "\n"				\
"1:	ldxr	%0, %2\n"						\
"	" #asm_op "	%0, %0, %3\n"					\
"	stxr	%w1, %0, %2\n"						\
"	cbnz	%w1, 1b"						\
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
	: "Ir" (i));							\
}									\
__LL_SC_EXPORT(atomic64_##op);

#define ATOMIC64_OP_RETURN(op, asm_op)					\
__LL_SC_INLINE long							\
__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v))		\
{									\
	long result;							\
	unsigned long tmp;						\
									\
	asm volatile("// atomic64_" #op "_return\n"			\
"1:	ldxr	%0, %2\n"						\
"	" #asm_op "	%0, %0, %3\n"					\
"	stlxr	%w1, %0, %2\n"						\
"	cbnz	%w1, 1b"						\
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
	: "Ir" (i)							\
	: "memory");							\
									\
	smp_mb();							\
	return result;							\
}									\
__LL_SC_EXPORT(atomic64_##op##_return);

#define ATOMIC64_OPS(op, asm_op)					\
	ATOMIC64_OP(op, asm_op)						\
	ATOMIC64_OP_RETURN(op, asm_op)

ATOMIC64_OPS(add, add)
ATOMIC64_OPS(sub, sub)

ATOMIC64_OP(and, and)
ATOMIC64_OP(andnot, bic)
ATOMIC64_OP(or, orr)
ATOMIC64_OP(xor, eor)

#undef ATOMIC64_OPS
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP

__LL_SC_INLINE long
__LL_SC_PREFIX(atomic64_cmpxchg(atomic64_t *ptr, long old, long new))
{
	long oldval;
	unsigned long res;

	smp_mb();

	asm volatile("// atomic64_cmpxchg\n"
"1:	ldxr	%1, %2\n"
"	cmp	%1, %3\n"
"	b.ne	2f\n"
"	stxr	%w0, %4, %2\n"
"	cbnz	%w0, 1b\n"
"2:"
	: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
	: "Ir" (old), "r" (new)
	: "cc");

	smp_mb();
	return oldval;
}
__LL_SC_EXPORT(atomic64_cmpxchg);

__LL_SC_INLINE long
__LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
{
	long result;
	unsigned long tmp;

	asm volatile("// atomic64_dec_if_positive\n"
"1:	ldxr	%0, %2\n"
"	subs	%0, %0, #1\n"
"	b.mi	2f\n"
"	stlxr	%w1, %0, %2\n"
"	cbnz	%w1, 1b\n"
"	dmb	ish\n"
"2:"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	:
	: "cc", "memory");

	return result;
}
__LL_SC_EXPORT(atomic64_dec_if_positive);

#define __CMPXCHG_CASE(w, sz, name, mb, cl)				\
__LL_SC_INLINE unsigned long						\
__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr,		\
				     unsigned long old,			\
				     unsigned long new))		\
{									\
	unsigned long tmp, oldval;					\
									\
	asm volatile(							\
	"	" #mb "\n"						\
	"1:	ldxr" #sz "\t%" #w "[oldval], %[v]\n"			\
	"	eor	%" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"	\
	"	cbnz	%" #w "[tmp], 2f\n"				\
	"	stxr" #sz "\t%w[tmp], %" #w "[new], %[v]\n"		\
	"	cbnz	%w[tmp], 1b\n"					\
	"	" #mb "\n"						\
	"	mov	%" #w "[oldval], %" #w "[old]\n"		\
	"2:"								\
	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),			\
	  [v] "+Q" (*(unsigned long *)ptr)				\
	: [old] "Lr" (old), [new] "r" (new)				\
	: cl);								\
									\
	return oldval;							\
}									\
__LL_SC_EXPORT(__cmpxchg_case_##name);

__CMPXCHG_CASE(w, b,    1,        ,         )
__CMPXCHG_CASE(w, h,    2,        ,         )
__CMPXCHG_CASE(w,  ,    4,        ,         )
__CMPXCHG_CASE( ,  ,    8,        ,         )
__CMPXCHG_CASE(w, b, mb_1, dmb ish, "memory")
__CMPXCHG_CASE(w, h, mb_2, dmb ish, "memory")
__CMPXCHG_CASE(w,  , mb_4, dmb ish, "memory")
__CMPXCHG_CASE( ,  , mb_8, dmb ish, "memory")

#undef __CMPXCHG_CASE

#endif	/* __ASM_ATOMIC_LL_SC_H */
Commit	Line	Data
c275f76b WD	1	/*
	2	* Based on arch/arm/include/asm/atomic.h
	3	*
	4	* Copyright (C) 1996 Russell King.
	5	* Copyright (C) 2002 Deep Blue Solutions Ltd.
	6	* Copyright (C) 2012 ARM Ltd.
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*
	12	* This program is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	* GNU General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU General Public License
	18	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#ifndef __ASM_ATOMIC_LL_SC_H
	22	#define __ASM_ATOMIC_LL_SC_H
	23
c0385b24 WD	24	#ifndef __ARM64_IN_ATOMIC_IMPL
	25	#error "please don't include this file directly"
	26	#endif
	27
c275f76b WD	28	/*
	29	* AArch64 UP and SMP safe atomic ops. We use load exclusive and
	30	* store exclusive to ensure that these are atomic. We may loop
	31	* to ensure that the update happens.
	32	*
	33	* NOTE: these functions do not follow the PCS and must explicitly
	34	* save any clobbered registers other than x0 (regardless of return
	35	* value). This is achieved through -fcall-saved-* compiler flags for
	36	* this file, which unfortunately don't work on a per-function basis
	37	* (the optimize attribute silently ignores these options).
	38	*/
	39
c275f76b WD	40	#define ATOMIC_OP(op, asm_op) \
	41	__LL_SC_INLINE void \
	42	__LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \
	43	{ \
	44	unsigned long tmp; \
	45	int result; \
	46	\
	47	asm volatile("// atomic_" #op "\n" \
	48	"1: ldxr %w0, %2\n" \
	49	" " #asm_op " %w0, %w0, %w3\n" \
	50	" stxr %w1, %w0, %2\n" \
	51	" cbnz %w1, 1b" \
	52	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	53	: "Ir" (i)); \
	54	} \
c0385b24	55	__LL_SC_EXPORT(atomic_##op);
c275f76b WD	56
	57	#define ATOMIC_OP_RETURN(op, asm_op) \
	58	__LL_SC_INLINE int \
	59	__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v)) \
	60	{ \
	61	unsigned long tmp; \
	62	int result; \
	63	\
	64	asm volatile("// atomic_" #op "_return\n" \
	65	"1: ldxr %w0, %2\n" \
	66	" " #asm_op " %w0, %w0, %w3\n" \
	67	" stlxr %w1, %w0, %2\n" \
	68	" cbnz %w1, 1b" \
	69	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	70	: "Ir" (i) \
	71	: "memory"); \
	72	\
	73	smp_mb(); \
	74	return result; \
c0385b24 WD	75	} \
c0385b24 WD	76	__LL_SC_EXPORT(atomic_##op##_return);
c275f76b WD	77
	78	#define ATOMIC_OPS(op, asm_op) \
	79	ATOMIC_OP(op, asm_op) \
	80	ATOMIC_OP_RETURN(op, asm_op)
	81
	82	ATOMIC_OPS(add, add)
	83	ATOMIC_OPS(sub, sub)
	84
	85	ATOMIC_OP(and, and)
	86	ATOMIC_OP(andnot, bic)
	87	ATOMIC_OP(or, orr)
	88	ATOMIC_OP(xor, eor)
	89
	90	#undef ATOMIC_OPS
	91	#undef ATOMIC_OP_RETURN
	92	#undef ATOMIC_OP
	93
	94	__LL_SC_INLINE int
	95	__LL_SC_PREFIX(atomic_cmpxchg(atomic_t *ptr, int old, int new))
	96	{
	97	unsigned long tmp;
	98	int oldval;
	99
	100	smp_mb();
	101
	102	asm volatile("// atomic_cmpxchg\n"
	103	"1: ldxr %w1, %2\n"
	104	" cmp %w1, %w3\n"
	105	" b.ne 2f\n"
	106	" stxr %w0, %w4, %2\n"
	107	" cbnz %w0, 1b\n"
	108	"2:"
	109	: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
	110	: "Ir" (old), "r" (new)
	111	: "cc");
	112
	113	smp_mb();
	114	return oldval;
	115	}
c0385b24	116	__LL_SC_EXPORT(atomic_cmpxchg);
c275f76b WD	117
	118	#define ATOMIC64_OP(op, asm_op) \
	119	__LL_SC_INLINE void \
	120	__LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \
	121	{ \
	122	long result; \
	123	unsigned long tmp; \
	124	\
	125	asm volatile("// atomic64_" #op "\n" \
	126	"1: ldxr %0, %2\n" \
	127	" " #asm_op " %0, %0, %3\n" \
	128	" stxr %w1, %0, %2\n" \
	129	" cbnz %w1, 1b" \
	130	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	131	: "Ir" (i)); \
	132	} \
c0385b24	133	__LL_SC_EXPORT(atomic64_##op);
c275f76b WD	134
	135	#define ATOMIC64_OP_RETURN(op, asm_op) \
	136	__LL_SC_INLINE long \
	137	__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v)) \
	138	{ \
	139	long result; \
	140	unsigned long tmp; \
	141	\
	142	asm volatile("// atomic64_" #op "_return\n" \
	143	"1: ldxr %0, %2\n" \
	144	" " #asm_op " %0, %0, %3\n" \
	145	" stlxr %w1, %0, %2\n" \
	146	" cbnz %w1, 1b" \
	147	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	148	: "Ir" (i) \
	149	: "memory"); \
	150	\
	151	smp_mb(); \
	152	return result; \
c0385b24 WD	153	} \
c0385b24 WD	154	__LL_SC_EXPORT(atomic64_##op##_return);
c275f76b WD	155
	156	#define ATOMIC64_OPS(op, asm_op) \
	157	ATOMIC64_OP(op, asm_op) \
	158	ATOMIC64_OP_RETURN(op, asm_op)
	159
	160	ATOMIC64_OPS(add, add)
	161	ATOMIC64_OPS(sub, sub)
	162
	163	ATOMIC64_OP(and, and)
	164	ATOMIC64_OP(andnot, bic)
	165	ATOMIC64_OP(or, orr)
	166	ATOMIC64_OP(xor, eor)
	167
	168	#undef ATOMIC64_OPS
	169	#undef ATOMIC64_OP_RETURN
	170	#undef ATOMIC64_OP
	171
	172	__LL_SC_INLINE long
	173	__LL_SC_PREFIX(atomic64_cmpxchg(atomic64_t *ptr, long old, long new))
	174	{
	175	long oldval;
	176	unsigned long res;
	177
	178	smp_mb();
	179
	180	asm volatile("// atomic64_cmpxchg\n"
	181	"1: ldxr %1, %2\n"
	182	" cmp %1, %3\n"
	183	" b.ne 2f\n"
	184	" stxr %w0, %4, %2\n"
	185	" cbnz %w0, 1b\n"
	186	"2:"
	187	: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
	188	: "Ir" (old), "r" (new)
	189	: "cc");
	190
	191	smp_mb();
	192	return oldval;
	193	}
c0385b24	194	__LL_SC_EXPORT(atomic64_cmpxchg);
c275f76b WD	195
	196	__LL_SC_INLINE long
	197	__LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
	198	{
	199	long result;
	200	unsigned long tmp;
	201
	202	asm volatile("// atomic64_dec_if_positive\n"
	203	"1: ldxr %0, %2\n"
	204	" subs %0, %0, #1\n"
	205	" b.mi 2f\n"
	206	" stlxr %w1, %0, %2\n"
	207	" cbnz %w1, 1b\n"
	208	" dmb ish\n"
	209	"2:"
	210	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	211	:
	212	: "cc", "memory");
	213
	214	return result;
	215	}
c0385b24	216	__LL_SC_EXPORT(atomic64_dec_if_positive);
c275f76b	217
c342f782 WD	218	#define __CMPXCHG_CASE(w, sz, name, mb, cl) \
	219	__LL_SC_INLINE unsigned long \
	220	__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
	221	unsigned long old, \
	222	unsigned long new)) \
	223	{ \
	224	unsigned long tmp, oldval; \
	225	\
	226	asm volatile( \
	227	" " #mb "\n" \
	228	"1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \
	229	" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
	230	" cbnz %" #w "[tmp], 2f\n" \
	231	" stxr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
	232	" cbnz %w[tmp], 1b\n" \
	233	" " #mb "\n" \
	234	" mov %" #w "[oldval], %" #w "[old]\n" \
	235	"2:" \
	236	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
	237	[v] "+Q" ((unsigned long )ptr) \
	238	: [old] "Lr" (old), [new] "r" (new) \
	239	: cl); \
	240	\
	241	return oldval; \
	242	} \
	243	__LL_SC_EXPORT(__cmpxchg_case_##name);
	244
	245	__CMPXCHG_CASE(w, b, 1, , )
	246	__CMPXCHG_CASE(w, h, 2, , )
	247	__CMPXCHG_CASE(w, , 4, , )
	248	__CMPXCHG_CASE( , , 8, , )
	249	__CMPXCHG_CASE(w, b, mb_1, dmb ish, "memory")
	250	__CMPXCHG_CASE(w, h, mb_2, dmb ish, "memory")
	251	__CMPXCHG_CASE(w, , mb_4, dmb ish, "memory")
	252	__CMPXCHG_CASE( , , mb_8, dmb ish, "memory")
	253
	254	#undef __CMPXCHG_CASE
	255
c275f76b	256	#endif /* __ASM_ATOMIC_LL_SC_H */