[deliverable/linux.git] / arch / arm64 / include / asm / atomic.h

/*
 * Based on arch/arm/include/asm/atomic.h
 *
 * Copyright (C) 1996 Russell King.
 * Copyright (C) 2002 Deep Blue Solutions Ltd.
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_ATOMIC_H
#define __ASM_ATOMIC_H

#include <linux/compiler.h>
#include <linux/types.h>

#include <asm/barrier.h>
#include <asm/cmpxchg.h>

#define ATOMIC_INIT(i)	{ (i) }

#ifdef __KERNEL__

/*
 * On ARM, ordinary assignment (str instruction) doesn't clear the local
 * strex/ldrex monitor on some implementations. The reason we can use it for
 * atomic_set() is the clrex or dummy strex done on every exception return.
 */
#define atomic_read(v)	(*(volatile int *)&(v)->counter)
#define atomic_set(v,i)	(((v)->counter) = (i))

/*
 * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
 * store exclusive to ensure that these are atomic.  We may loop
 * to ensure that the update happens.
 */
static inline void atomic_add(int i, atomic_t *v)
{
	unsigned long tmp;
	int result;

	asm volatile("// atomic_add\n"
"1:	ldxr	%w0, %2\n"
"	add	%w0, %w0, %w3\n"
"	stxr	%w1, %w0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i));
}

static inline int atomic_add_return(int i, atomic_t *v)
{
	unsigned long tmp;
	int result;

	asm volatile("// atomic_add_return\n"
"1:	ldxr	%w0, %2\n"
"	add	%w0, %w0, %w3\n"
"	stlxr	%w1, %w0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i)
	: "memory");

	smp_mb();
	return result;
}

static inline void atomic_sub(int i, atomic_t *v)
{
	unsigned long tmp;
	int result;

	asm volatile("// atomic_sub\n"
"1:	ldxr	%w0, %2\n"
"	sub	%w0, %w0, %w3\n"
"	stxr	%w1, %w0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i));
}

static inline int atomic_sub_return(int i, atomic_t *v)
{
	unsigned long tmp;
	int result;

	asm volatile("// atomic_sub_return\n"
"1:	ldxr	%w0, %2\n"
"	sub	%w0, %w0, %w3\n"
"	stlxr	%w1, %w0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i)
	: "memory");

	smp_mb();
	return result;
}

static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
{
	unsigned long tmp;
	int oldval;

	smp_mb();

	asm volatile("// atomic_cmpxchg\n"
"1:	ldxr	%w1, %2\n"
"	cmp	%w1, %w3\n"
"	b.ne	2f\n"
"	stxr	%w0, %w4, %2\n"
"	cbnz	%w0, 1b\n"
"2:"
	: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
	: "Ir" (old), "r" (new)
	: "cc");

	smp_mb();
	return oldval;
}

#define atomic_xchg(v, new) (xchg(&((v)->counter), new))

static inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
	int c, old;

	c = atomic_read(v);
	while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
		c = old;
	return c;
}

#define atomic_inc(v)		atomic_add(1, v)
#define atomic_dec(v)		atomic_sub(1, v)

#define atomic_inc_and_test(v)	(atomic_add_return(1, v) == 0)
#define atomic_dec_and_test(v)	(atomic_sub_return(1, v) == 0)
#define atomic_inc_return(v)    (atomic_add_return(1, v))
#define atomic_dec_return(v)    (atomic_sub_return(1, v))
#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)

#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)

#define smp_mb__before_atomic_dec()	smp_mb()
#define smp_mb__after_atomic_dec()	smp_mb()
#define smp_mb__before_atomic_inc()	smp_mb()
#define smp_mb__after_atomic_inc()	smp_mb()

/*
 * 64-bit atomic operations.
 */
#define ATOMIC64_INIT(i) { (i) }

#define atomic64_read(v)	(*(volatile long long *)&(v)->counter)
#define atomic64_set(v,i)	(((v)->counter) = (i))

static inline void atomic64_add(u64 i, atomic64_t *v)
{
	long result;
	unsigned long tmp;

	asm volatile("// atomic64_add\n"
"1:	ldxr	%0, %2\n"
"	add	%0, %0, %3\n"
"	stxr	%w1, %0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i));
}

static inline long atomic64_add_return(long i, atomic64_t *v)
{
	long result;
	unsigned long tmp;

	asm volatile("// atomic64_add_return\n"
"1:	ldxr	%0, %2\n"
"	add	%0, %0, %3\n"
"	stlxr	%w1, %0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i)
	: "memory");

	smp_mb();
	return result;
}

static inline void atomic64_sub(u64 i, atomic64_t *v)
{
	long result;
	unsigned long tmp;

	asm volatile("// atomic64_sub\n"
"1:	ldxr	%0, %2\n"
"	sub	%0, %0, %3\n"
"	stxr	%w1, %0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i));
}

static inline long atomic64_sub_return(long i, atomic64_t *v)
{
	long result;
	unsigned long tmp;

	asm volatile("// atomic64_sub_return\n"
"1:	ldxr	%0, %2\n"
"	sub	%0, %0, %3\n"
"	stlxr	%w1, %0, %2\n"
"	cbnz	%w1, 1b"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	: "Ir" (i)
	: "memory");

	smp_mb();
	return result;
}

static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
{
	long oldval;
	unsigned long res;

	smp_mb();

	asm volatile("// atomic64_cmpxchg\n"
"1:	ldxr	%1, %2\n"
"	cmp	%1, %3\n"
"	b.ne	2f\n"
"	stxr	%w0, %4, %2\n"
"	cbnz	%w0, 1b\n"
"2:"
	: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
	: "Ir" (old), "r" (new)
	: "cc");

	smp_mb();
	return oldval;
}

#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))

static inline long atomic64_dec_if_positive(atomic64_t *v)
{
	long result;
	unsigned long tmp;

	asm volatile("// atomic64_dec_if_positive\n"
"1:	ldxr	%0, %2\n"
"	subs	%0, %0, #1\n"
"	b.mi	2f\n"
"	stlxr	%w1, %0, %2\n"
"	cbnz	%w1, 1b\n"
"	dmb	ish\n"
"2:"
	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	:
	: "cc", "memory");

	return result;
}

static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
{
	long c, old;

	c = atomic64_read(v);
	while (c != u && (old = atomic64_cmpxchg((v), c, c + a)) != c)
		c = old;

	return c != u;
}

#define atomic64_add_negative(a, v)	(atomic64_add_return((a), (v)) < 0)
#define atomic64_inc(v)			atomic64_add(1LL, (v))
#define atomic64_inc_return(v)		atomic64_add_return(1LL, (v))
#define atomic64_inc_and_test(v)	(atomic64_inc_return(v) == 0)
#define atomic64_sub_and_test(a, v)	(atomic64_sub_return((a), (v)) == 0)
#define atomic64_dec(v)			atomic64_sub(1LL, (v))
#define atomic64_dec_return(v)		atomic64_sub_return(1LL, (v))
#define atomic64_dec_and_test(v)	(atomic64_dec_return((v)) == 0)
#define atomic64_inc_not_zero(v)	atomic64_add_unless((v), 1LL, 0LL)

#endif
#endif
Commit	Line	Data
6170a974 CM	1	/*
	2	* Based on arch/arm/include/asm/atomic.h
	3	*
	4	* Copyright (C) 1996 Russell King.
	5	* Copyright (C) 2002 Deep Blue Solutions Ltd.
	6	* Copyright (C) 2012 ARM Ltd.
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*
	12	* This program is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	* GNU General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU General Public License
	18	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	*/
	20	#ifndef __ASM_ATOMIC_H
	21	#define __ASM_ATOMIC_H
	22
	23	#include <linux/compiler.h>
	24	#include <linux/types.h>
	25
	26	#include <asm/barrier.h>
	27	#include <asm/cmpxchg.h>
	28
	29	#define ATOMIC_INIT(i) { (i) }
	30
	31	#ifdef __KERNEL__
	32
	33	/*
	34	* On ARM, ordinary assignment (str instruction) doesn't clear the local
	35	* strex/ldrex monitor on some implementations. The reason we can use it for
	36	* atomic_set() is the clrex or dummy strex done on every exception return.
	37	*/
	38	#define atomic_read(v) ((volatile int )&(v)->counter)
	39	#define atomic_set(v,i) (((v)->counter) = (i))
	40
	41	/*
	42	* AArch64 UP and SMP safe atomic ops. We use load exclusive and
	43	* store exclusive to ensure that these are atomic. We may loop
	44	* to ensure that the update happens.
	45	*/
	46	static inline void atomic_add(int i, atomic_t *v)
	47	{
	48	unsigned long tmp;
	49	int result;
	50
	51	asm volatile("// atomic_add\n"
3a0310eb WD	52	"1: ldxr %w0, %2\n"
	53	" add %w0, %w0, %w3\n"
	54	" stxr %w1, %w0, %2\n"
6170a974	55	" cbnz %w1, 1b"
3a0310eb	56	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
95c41896	57	: "Ir" (i));
6170a974 CM	58	}
	59
	60	static inline int atomic_add_return(int i, atomic_t *v)
	61	{
	62	unsigned long tmp;
	63	int result;
	64
	65	asm volatile("// atomic_add_return\n"
8e86f0b4	66	"1: ldxr %w0, %2\n"
3a0310eb WD	67	" add %w0, %w0, %w3\n"
3a0310eb WD	68	" stlxr %w1, %w0, %2\n"
6170a974	69	" cbnz %w1, 1b"
3a0310eb WD	70	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
3a0310eb WD	71	: "Ir" (i)
95c41896	72	: "memory");
6170a974	73
8e86f0b4	74	smp_mb();
6170a974 CM	75	return result;
	76	}
	77
	78	static inline void atomic_sub(int i, atomic_t *v)
	79	{
	80	unsigned long tmp;
	81	int result;
	82
	83	asm volatile("// atomic_sub\n"
3a0310eb WD	84	"1: ldxr %w0, %2\n"
	85	" sub %w0, %w0, %w3\n"
	86	" stxr %w1, %w0, %2\n"
6170a974	87	" cbnz %w1, 1b"
3a0310eb	88	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
95c41896	89	: "Ir" (i));
6170a974 CM	90	}
	91
	92	static inline int atomic_sub_return(int i, atomic_t *v)
	93	{
	94	unsigned long tmp;
	95	int result;
	96
	97	asm volatile("// atomic_sub_return\n"
8e86f0b4	98	"1: ldxr %w0, %2\n"
3a0310eb WD	99	" sub %w0, %w0, %w3\n"
3a0310eb WD	100	" stlxr %w1, %w0, %2\n"
6170a974	101	" cbnz %w1, 1b"
3a0310eb WD	102	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
3a0310eb WD	103	: "Ir" (i)
95c41896	104	: "memory");
6170a974	105
8e86f0b4	106	smp_mb();
6170a974 CM	107	return result;
	108	}
	109
	110	static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
	111	{
	112	unsigned long tmp;
	113	int oldval;
	114
8e86f0b4 WD	115	smp_mb();
8e86f0b4 WD	116
6170a974	117	asm volatile("// atomic_cmpxchg\n"
8e86f0b4	118	"1: ldxr %w1, %2\n"
3a0310eb	119	" cmp %w1, %w3\n"
6170a974	120	" b.ne 2f\n"
8e86f0b4	121	" stxr %w0, %w4, %2\n"
6170a974 CM	122	" cbnz %w0, 1b\n"
6170a974 CM	123	"2:"
3a0310eb WD	124	: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
3a0310eb WD	125	: "Ir" (old), "r" (new)
95c41896	126	: "cc");
6170a974	127
8e86f0b4	128	smp_mb();
6170a974 CM	129	return oldval;
	130	}
	131
6170a974 CM	132	#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
	133
	134	static inline int __atomic_add_unless(atomic_t *v, int a, int u)
	135	{
	136	int c, old;
	137
	138	c = atomic_read(v);
	139	while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
	140	c = old;
	141	return c;
	142	}
	143
	144	#define atomic_inc(v) atomic_add(1, v)
	145	#define atomic_dec(v) atomic_sub(1, v)
	146
	147	#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)
	148	#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
	149	#define atomic_inc_return(v) (atomic_add_return(1, v))
	150	#define atomic_dec_return(v) (atomic_sub_return(1, v))
	151	#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
	152
	153	#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)
	154
	155	#define smp_mb__before_atomic_dec() smp_mb()
	156	#define smp_mb__after_atomic_dec() smp_mb()
	157	#define smp_mb__before_atomic_inc() smp_mb()
	158	#define smp_mb__after_atomic_inc() smp_mb()
	159
	160	/*
	161	* 64-bit atomic operations.
	162	*/
	163	#define ATOMIC64_INIT(i) { (i) }
	164
	165	#define atomic64_read(v) ((volatile long long )&(v)->counter)
	166	#define atomic64_set(v,i) (((v)->counter) = (i))
	167
	168	static inline void atomic64_add(u64 i, atomic64_t *v)
	169	{
	170	long result;
	171	unsigned long tmp;
	172
	173	asm volatile("// atomic64_add\n"
3a0310eb WD	174	"1: ldxr %0, %2\n"
	175	" add %0, %0, %3\n"
	176	" stxr %w1, %0, %2\n"
6170a974	177	" cbnz %w1, 1b"
3a0310eb	178	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
95c41896	179	: "Ir" (i));
6170a974 CM	180	}
	181
	182	static inline long atomic64_add_return(long i, atomic64_t *v)
	183	{
	184	long result;
	185	unsigned long tmp;
	186
	187	asm volatile("// atomic64_add_return\n"
8e86f0b4	188	"1: ldxr %0, %2\n"
3a0310eb WD	189	" add %0, %0, %3\n"
3a0310eb WD	190	" stlxr %w1, %0, %2\n"
6170a974	191	" cbnz %w1, 1b"
3a0310eb WD	192	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
3a0310eb WD	193	: "Ir" (i)
95c41896	194	: "memory");
6170a974	195
8e86f0b4	196	smp_mb();
6170a974 CM	197	return result;
	198	}
	199
	200	static inline void atomic64_sub(u64 i, atomic64_t *v)
	201	{
	202	long result;
	203	unsigned long tmp;
	204
	205	asm volatile("// atomic64_sub\n"
3a0310eb WD	206	"1: ldxr %0, %2\n"
	207	" sub %0, %0, %3\n"
	208	" stxr %w1, %0, %2\n"
6170a974	209	" cbnz %w1, 1b"
3a0310eb	210	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
95c41896	211	: "Ir" (i));
6170a974 CM	212	}
	213
	214	static inline long atomic64_sub_return(long i, atomic64_t *v)
	215	{
	216	long result;
	217	unsigned long tmp;
	218
	219	asm volatile("// atomic64_sub_return\n"
8e86f0b4	220	"1: ldxr %0, %2\n"
3a0310eb WD	221	" sub %0, %0, %3\n"
3a0310eb WD	222	" stlxr %w1, %0, %2\n"
6170a974	223	" cbnz %w1, 1b"
3a0310eb WD	224	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
3a0310eb WD	225	: "Ir" (i)
95c41896	226	: "memory");
6170a974	227
8e86f0b4	228	smp_mb();
6170a974 CM	229	return result;
	230	}
	231
	232	static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
	233	{
	234	long oldval;
	235	unsigned long res;
	236
8e86f0b4 WD	237	smp_mb();
8e86f0b4 WD	238
6170a974	239	asm volatile("// atomic64_cmpxchg\n"
8e86f0b4	240	"1: ldxr %1, %2\n"
3a0310eb	241	" cmp %1, %3\n"
6170a974	242	" b.ne 2f\n"
8e86f0b4	243	" stxr %w0, %4, %2\n"
6170a974 CM	244	" cbnz %w0, 1b\n"
6170a974 CM	245	"2:"
3a0310eb WD	246	: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
3a0310eb WD	247	: "Ir" (old), "r" (new)
95c41896	248	: "cc");
6170a974	249
8e86f0b4	250	smp_mb();
6170a974 CM	251	return oldval;
	252	}
	253
	254	#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
	255
	256	static inline long atomic64_dec_if_positive(atomic64_t *v)
	257	{
	258	long result;
	259	unsigned long tmp;
	260
	261	asm volatile("// atomic64_dec_if_positive\n"
8e86f0b4	262	"1: ldxr %0, %2\n"
6170a974 CM	263	" subs %0, %0, #1\n"
6170a974 CM	264	" b.mi 2f\n"
3a0310eb	265	" stlxr %w1, %0, %2\n"
6170a974	266	" cbnz %w1, 1b\n"
8e86f0b4	267	" dmb ish\n"
6170a974	268	"2:"
3a0310eb WD	269	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	270	:
	271	: "cc", "memory");
6170a974 CM	272
	273	return result;
	274	}
	275
	276	static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
	277	{
	278	long c, old;
	279
	280	c = atomic64_read(v);
	281	while (c != u && (old = atomic64_cmpxchg((v), c, c + a)) != c)
	282	c = old;
	283
	284	return c != u;
	285	}
	286
	287	#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
	288	#define atomic64_inc(v) atomic64_add(1LL, (v))
	289	#define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
	290	#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
	291	#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
	292	#define atomic64_dec(v) atomic64_sub(1LL, (v))
	293	#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))
	294	#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
	295	#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
	296
	297	#endif
	298	#endif