[deliverable/linux.git] / arch / arm64 / include / asm / cmpxchg.h

/*
 * Based on arch/arm/include/asm/cmpxchg.h
 *
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_CMPXCHG_H
#define __ASM_CMPXCHG_H

#include <linux/bug.h>

#include <asm/barrier.h>

static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
{
	unsigned long ret, tmp;

	switch (size) {
	case 1:
		asm volatile("//	__xchg1\n"
		"1:	ldxrb	%w0, %2\n"
		"	stlxrb	%w1, %w3, %2\n"
		"	cbnz	%w1, 1b\n"
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
			: "r" (x)
			: "memory");
		break;
	case 2:
		asm volatile("//	__xchg2\n"
		"1:	ldxrh	%w0, %2\n"
		"	stlxrh	%w1, %w3, %2\n"
		"	cbnz	%w1, 1b\n"
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
			: "r" (x)
			: "memory");
		break;
	case 4:
		asm volatile("//	__xchg4\n"
		"1:	ldxr	%w0, %2\n"
		"	stlxr	%w1, %w3, %2\n"
		"	cbnz	%w1, 1b\n"
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
			: "r" (x)
			: "memory");
		break;
	case 8:
		asm volatile("//	__xchg8\n"
		"1:	ldxr	%0, %2\n"
		"	stlxr	%w1, %3, %2\n"
		"	cbnz	%w1, 1b\n"
			: "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
			: "r" (x)
			: "memory");
		break;
	default:
		BUILD_BUG();
	}

	smp_mb();
	return ret;
}

#define xchg(ptr,x) \
	((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))

static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
				      unsigned long new, int size)
{
	unsigned long oldval = 0, res;

	switch (size) {
	case 1:
		do {
			asm volatile("// __cmpxchg1\n"
			"	ldxrb	%w1, %2\n"
			"	mov	%w0, #0\n"
			"	cmp	%w1, %w3\n"
			"	b.ne	1f\n"
			"	stxrb	%w0, %w4, %2\n"
			"1:\n"
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr)
				: "Ir" (old), "r" (new)
				: "cc");
		} while (res);
		break;

	case 2:
		do {
			asm volatile("// __cmpxchg2\n"
			"	ldxrh	%w1, %2\n"
			"	mov	%w0, #0\n"
			"	cmp	%w1, %w3\n"
			"	b.ne	1f\n"
			"	stxrh	%w0, %w4, %2\n"
			"1:\n"
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr)
				: "Ir" (old), "r" (new)
				: "cc");
		} while (res);
		break;

	case 4:
		do {
			asm volatile("// __cmpxchg4\n"
			"	ldxr	%w1, %2\n"
			"	mov	%w0, #0\n"
			"	cmp	%w1, %w3\n"
			"	b.ne	1f\n"
			"	stxr	%w0, %w4, %2\n"
			"1:\n"
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr)
				: "Ir" (old), "r" (new)
				: "cc");
		} while (res);
		break;

	case 8:
		do {
			asm volatile("// __cmpxchg8\n"
			"	ldxr	%1, %2\n"
			"	mov	%w0, #0\n"
			"	cmp	%1, %3\n"
			"	b.ne	1f\n"
			"	stxr	%w0, %4, %2\n"
			"1:\n"
				: "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr)
				: "Ir" (old), "r" (new)
				: "cc");
		} while (res);
		break;

	default:
		BUILD_BUG();
	}

	return oldval;
}

static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
					 unsigned long new, int size)
{
	unsigned long ret;

	smp_mb();
	ret = __cmpxchg(ptr, old, new, size);
	smp_mb();

	return ret;
}

#define cmpxchg(ptr, o, n) \
({ \
	__typeof__(*(ptr)) __ret; \
	__ret = (__typeof__(*(ptr))) \
		__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
			     sizeof(*(ptr))); \
	__ret; \
})

#define cmpxchg_local(ptr, o, n) \
({ \
	__typeof__(*(ptr)) __ret; \
	__ret = (__typeof__(*(ptr))) \
		__cmpxchg((ptr), (unsigned long)(o), \
			  (unsigned long)(n), sizeof(*(ptr))); \
	__ret; \
})

#define cmpxchg64(ptr,o,n)		cmpxchg((ptr),(o),(n))
#define cmpxchg64_local(ptr,o,n)	cmpxchg_local((ptr),(o),(n))

#define cmpxchg64_relaxed(ptr,o,n)	cmpxchg_local((ptr),(o),(n))

#endif	/* __ASM_CMPXCHG_H */
Commit	Line	Data
10b663ae CM	1	/*
	2	* Based on arch/arm/include/asm/cmpxchg.h
	3	*
	4	* Copyright (C) 2012 ARM Ltd.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18	#ifndef __ASM_CMPXCHG_H
	19	#define __ASM_CMPXCHG_H
	20
	21	#include <linux/bug.h>
	22
	23	#include <asm/barrier.h>
	24
	25	static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
	26	{
	27	unsigned long ret, tmp;
	28
	29	switch (size) {
	30	case 1:
	31	asm volatile("// __xchg1\n"
8e86f0b4	32	"1: ldxrb %w0, %2\n"
3a0310eb	33	" stlxrb %w1, %w3, %2\n"
10b663ae	34	" cbnz %w1, 1b\n"
3a0310eb WD	35	: "=&r" (ret), "=&r" (tmp), "+Q" ((u8 )ptr)
3a0310eb WD	36	: "r" (x)
95c41896	37	: "memory");
10b663ae CM	38	break;
	39	case 2:
	40	asm volatile("// __xchg2\n"
8e86f0b4	41	"1: ldxrh %w0, %2\n"
3a0310eb	42	" stlxrh %w1, %w3, %2\n"
10b663ae	43	" cbnz %w1, 1b\n"
3a0310eb WD	44	: "=&r" (ret), "=&r" (tmp), "+Q" ((u16 )ptr)
3a0310eb WD	45	: "r" (x)
95c41896	46	: "memory");
10b663ae CM	47	break;
	48	case 4:
	49	asm volatile("// __xchg4\n"
8e86f0b4	50	"1: ldxr %w0, %2\n"
3a0310eb	51	" stlxr %w1, %w3, %2\n"
10b663ae	52	" cbnz %w1, 1b\n"
3a0310eb WD	53	: "=&r" (ret), "=&r" (tmp), "+Q" ((u32 )ptr)
3a0310eb WD	54	: "r" (x)
95c41896	55	: "memory");
10b663ae CM	56	break;
	57	case 8:
	58	asm volatile("// __xchg8\n"
8e86f0b4	59	"1: ldxr %0, %2\n"
3a0310eb	60	" stlxr %w1, %3, %2\n"
10b663ae	61	" cbnz %w1, 1b\n"
3a0310eb WD	62	: "=&r" (ret), "=&r" (tmp), "+Q" ((u64 )ptr)
3a0310eb WD	63	: "r" (x)
95c41896	64	: "memory");
10b663ae CM	65	break;
	66	default:
	67	BUILD_BUG();
	68	}
	69
8e86f0b4	70	smp_mb();
10b663ae CM	71	return ret;
	72	}
	73
	74	#define xchg(ptr,x) \
	75	((__typeof__((ptr)))__xchg((unsigned long)(x),(ptr),sizeof((ptr))))
	76
	77	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
	78	unsigned long new, int size)
	79	{
	80	unsigned long oldval = 0, res;
	81
	82	switch (size) {
	83	case 1:
	84	do {
	85	asm volatile("// __cmpxchg1\n"
3a0310eb	86	" ldxrb %w1, %2\n"
10b663ae CM	87	" mov %w0, #0\n"
	88	" cmp %w1, %w3\n"
	89	" b.ne 1f\n"
3a0310eb	90	" stxrb %w0, %w4, %2\n"
10b663ae	91	"1:\n"
3a0310eb WD	92	: "=&r" (res), "=&r" (oldval), "+Q" ((u8 )ptr)
3a0310eb WD	93	: "Ir" (old), "r" (new)
10b663ae CM	94	: "cc");
	95	} while (res);
	96	break;
	97
	98	case 2:
	99	do {
	100	asm volatile("// __cmpxchg2\n"
3a0310eb	101	" ldxrh %w1, %2\n"
10b663ae CM	102	" mov %w0, #0\n"
	103	" cmp %w1, %w3\n"
	104	" b.ne 1f\n"
3a0310eb	105	" stxrh %w0, %w4, %2\n"
10b663ae	106	"1:\n"
3a0310eb WD	107	: "=&r" (res), "=&r" (oldval), "+Q" ((u16 )ptr)
	108	: "Ir" (old), "r" (new)
	109	: "cc");
10b663ae CM	110	} while (res);
	111	break;
	112
	113	case 4:
	114	do {
	115	asm volatile("// __cmpxchg4\n"
3a0310eb	116	" ldxr %w1, %2\n"
10b663ae CM	117	" mov %w0, #0\n"
	118	" cmp %w1, %w3\n"
	119	" b.ne 1f\n"
3a0310eb	120	" stxr %w0, %w4, %2\n"
10b663ae	121	"1:\n"
3a0310eb WD	122	: "=&r" (res), "=&r" (oldval), "+Q" ((u32 )ptr)
3a0310eb WD	123	: "Ir" (old), "r" (new)
10b663ae CM	124	: "cc");
	125	} while (res);
	126	break;
	127
	128	case 8:
	129	do {
	130	asm volatile("// __cmpxchg8\n"
3a0310eb	131	" ldxr %1, %2\n"
10b663ae CM	132	" mov %w0, #0\n"
	133	" cmp %1, %3\n"
	134	" b.ne 1f\n"
3a0310eb	135	" stxr %w0, %4, %2\n"
10b663ae	136	"1:\n"
3a0310eb WD	137	: "=&r" (res), "=&r" (oldval), "+Q" ((u64 )ptr)
3a0310eb WD	138	: "Ir" (old), "r" (new)
10b663ae CM	139	: "cc");
	140	} while (res);
	141	break;
	142
	143	default:
	144	BUILD_BUG();
	145	}
	146
	147	return oldval;
	148	}
	149
	150	static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
	151	unsigned long new, int size)
	152	{
	153	unsigned long ret;
	154
	155	smp_mb();
	156	ret = __cmpxchg(ptr, old, new, size);
	157	smp_mb();
	158
	159	return ret;
	160	}
	161
60010e50 MH	162	#define cmpxchg(ptr, o, n) \
	163	({ \
	164	__typeof__(*(ptr)) __ret; \
	165	__ret = (__typeof__(*(ptr))) \
	166	__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
	167	sizeof(*(ptr))); \
	168	__ret; \
	169	})
	170
	171	#define cmpxchg_local(ptr, o, n) \
	172	({ \
	173	__typeof__(*(ptr)) __ret; \
	174	__ret = (__typeof__(*(ptr))) \
	175	__cmpxchg((ptr), (unsigned long)(o), \
	176	(unsigned long)(n), sizeof(*(ptr))); \
	177	__ret; \
	178	})
10b663ae	179
a84b086b CG	180	#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
	181	#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
	182
cf10b79a WD	183	#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n))
cf10b79a WD	184
10b663ae	185	#endif /* __ASM_CMPXCHG_H */