[deliverable/linux.git] / arch / x86 / include / asm / cmpxchg.h

#ifndef ASM_X86_CMPXCHG_H
#define ASM_X86_CMPXCHG_H

#include <linux/compiler.h>
#include <asm/alternative.h> /* Provides LOCK_PREFIX */

#define __HAVE_ARCH_CMPXCHG 1

/*
 * Non-existant functions to indicate usage errors at link time
 * (or compile-time if the compiler implements __compiletime_error().
 */
extern void __xchg_wrong_size(void)
	__compiletime_error("Bad argument size for xchg");
extern void __cmpxchg_wrong_size(void)
	__compiletime_error("Bad argument size for cmpxchg");
extern void __xadd_wrong_size(void)
	__compiletime_error("Bad argument size for xadd");
extern void __add_wrong_size(void)
	__compiletime_error("Bad argument size for add");

/*
 * Constants for operation sizes. On 32-bit, the 64-bit size it set to
 * -1 because sizeof will never return -1, thereby making those switch
 * case statements guaranteeed dead code which the compiler will
 * eliminate, and allowing the "missing symbol in the default case" to
 * indicate a usage error.
 */
#define __X86_CASE_B	1
#define __X86_CASE_W	2
#define __X86_CASE_L	4
#ifdef CONFIG_64BIT
#define __X86_CASE_Q	8
#else
#define	__X86_CASE_Q	-1		/* sizeof will never return -1 */
#endif

/* 
 * An exchange-type operation, which takes a value and a pointer, and
 * returns the old value.
 */
#define __xchg_op(ptr, arg, op, lock)					\
	({								\
	        __typeof__ (*(ptr)) __ret = (arg);			\
		switch (sizeof(*(ptr))) {				\
		case __X86_CASE_B:					\
			asm volatile (lock #op "b %b0, %1\n"		\
				      : "+q" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		case __X86_CASE_W:					\
			asm volatile (lock #op "w %w0, %1\n"		\
				      : "+r" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		case __X86_CASE_L:					\
			asm volatile (lock #op "l %0, %1\n"		\
				      : "+r" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		case __X86_CASE_Q:					\
			asm volatile (lock #op "q %q0, %1\n"		\
				      : "+r" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		default:						\
			__ ## op ## _wrong_size();			\
		}							\
		__ret;							\
	})

/*
 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
 * Since this is generally used to protect other memory information, we
 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
 * information around.
 */
#define xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")

/*
 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 * store NEW in MEM.  Return the initial value in MEM.  Success is
 * indicated by comparing RETURN with OLD.
 */
#define __raw_cmpxchg(ptr, old, new, size, lock)			\
({									\
	__typeof__(*(ptr)) __ret;					\
	__typeof__(*(ptr)) __old = (old);				\
	__typeof__(*(ptr)) __new = (new);				\
	switch (size) {							\
	case __X86_CASE_B:						\
	{								\
		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
		asm volatile(lock "cmpxchgb %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "q" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_W:						\
	{								\
		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
		asm volatile(lock "cmpxchgw %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_L:						\
	{								\
		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
		asm volatile(lock "cmpxchgl %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_Q:						\
	{								\
		volatile u64 *__ptr = (volatile u64 *)(ptr);		\
		asm volatile(lock "cmpxchgq %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	default:							\
		__cmpxchg_wrong_size();					\
	}								\
	__ret;								\
})

#define __cmpxchg(ptr, old, new, size)					\
	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)

#define __sync_cmpxchg(ptr, old, new, size)				\
	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")

#define __cmpxchg_local(ptr, old, new, size)				\
	__raw_cmpxchg((ptr), (old), (new), (size), "")

#ifdef CONFIG_X86_32
# include <asm/cmpxchg_32.h>
#else
# include <asm/cmpxchg_64.h>
#endif

#define cmpxchg(ptr, old, new)						\
	__cmpxchg(ptr, old, new, sizeof(*(ptr)))

#define sync_cmpxchg(ptr, old, new)					\
	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))

#define cmpxchg_local(ptr, old, new)					\
	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))

/*
 * xadd() adds "inc" to "*ptr" and atomically returns the previous
 * value of "*ptr".
 *
 * xadd() is locked when multiple CPUs are online
 * xadd_sync() is always locked
 * xadd_local() is never locked
 */
#define __xadd(ptr, inc, lock)	__xchg_op((ptr), (inc), xadd, lock)
#define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)
#define xadd_sync(ptr, inc)	__xadd((ptr), (inc), "lock; ")
#define xadd_local(ptr, inc)	__xadd((ptr), (inc), "")

#define __add(ptr, inc, lock)						\
	({								\
	        __typeof__ (*(ptr)) __ret = (inc);			\
		switch (sizeof(*(ptr))) {				\
		case __X86_CASE_B:					\
			asm volatile (lock "addb %b1, %0\n"		\
				      : "+m" (*(ptr)) : "qi" (inc)	\
				      : "memory", "cc");		\
			break;						\
		case __X86_CASE_W:					\
			asm volatile (lock "addw %w1, %0\n"		\
				      : "+m" (*(ptr)) : "ri" (inc)	\
				      : "memory", "cc");		\
			break;						\
		case __X86_CASE_L:					\
			asm volatile (lock "addl %1, %0\n"		\
				      : "+m" (*(ptr)) : "ri" (inc)	\
				      : "memory", "cc");		\
			break;						\
		case __X86_CASE_Q:					\
			asm volatile (lock "addq %1, %0\n"		\
				      : "+m" (*(ptr)) : "ri" (inc)	\
				      : "memory", "cc");		\
			break;						\
		default:						\
			__add_wrong_size();				\
		}							\
		__ret;							\
	})

/*
 * add_*() adds "inc" to "*ptr"
 *
 * __add() takes a lock prefix
 * add_smp() is locked when multiple CPUs are online
 * add_sync() is always locked
 */
#define add_smp(ptr, inc)	__add((ptr), (inc), LOCK_PREFIX)
#define add_sync(ptr, inc)	__add((ptr), (inc), "lock; ")

#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2)			\
({									\
	bool __ret;							\
	__typeof__(*(p1)) __old1 = (o1), __new1 = (n1);			\
	__typeof__(*(p2)) __old2 = (o2), __new2 = (n2);			\
	BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long));			\
	BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long));			\
	VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long)));		\
	VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2));	\
	asm volatile(pfx "cmpxchg%c4b %2; sete %0"			\
		     : "=a" (__ret), "+d" (__old2),			\
		       "+m" (*(p1)), "+m" (*(p2))			\
		     : "i" (2 * sizeof(long)), "a" (__old1),		\
		       "b" (__new1), "c" (__new2));			\
	__ret;								\
})

#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
	__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)

#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
	__cmpxchg_double(, p1, p2, o1, o2, n1, n2)

#endif	/* ASM_X86_CMPXCHG_H */
Commit	Line	Data
e9826380 JF	1	#ifndef ASM_X86_CMPXCHG_H
	2	#define ASM_X86_CMPXCHG_H
	3
61e2cd0a	4	#include <linux/compiler.h>
e9826380 JF	5	#include <asm/alternative.h> /* Provides LOCK_PREFIX */
e9826380 JF	6
b08ee5f7 BP	7	#define __HAVE_ARCH_CMPXCHG 1
b08ee5f7 BP	8
61e2cd0a JF	9	/*
	10	* Non-existant functions to indicate usage errors at link time
	11	* (or compile-time if the compiler implements __compiletime_error().
	12	*/
	13	extern void __xchg_wrong_size(void)
	14	__compiletime_error("Bad argument size for xchg");
	15	extern void __cmpxchg_wrong_size(void)
	16	__compiletime_error("Bad argument size for cmpxchg");
	17	extern void __xadd_wrong_size(void)
	18	__compiletime_error("Bad argument size for xadd");
3d94ae0c JF	19	extern void __add_wrong_size(void)
3d94ae0c JF	20	__compiletime_error("Bad argument size for add");
e9826380 JF	21
	22	/*
	23	* Constants for operation sizes. On 32-bit, the 64-bit size it set to
	24	* -1 because sizeof will never return -1, thereby making those switch
	25	* case statements guaranteeed dead code which the compiler will
	26	* eliminate, and allowing the "missing symbol in the default case" to
	27	* indicate a usage error.
	28	*/
	29	#define __X86_CASE_B 1
	30	#define __X86_CASE_W 2
	31	#define __X86_CASE_L 4
	32	#ifdef CONFIG_64BIT
	33	#define __X86_CASE_Q 8
	34	#else
	35	#define __X86_CASE_Q -1 /* sizeof will never return -1 */
	36	#endif
	37
31a8394e JF	38	/*
31a8394e JF	39	* An exchange-type operation, which takes a value and a pointer, and
7f5281ae	40	* returns the old value.
31a8394e JF	41	*/
	42	#define __xchg_op(ptr, arg, op, lock) \
	43	({ \
	44	__typeof__ (*(ptr)) __ret = (arg); \
	45	switch (sizeof(*(ptr))) { \
	46	case __X86_CASE_B: \
	47	asm volatile (lock #op "b %b0, %1\n" \
2ca052a3	48	: "+q" (__ret), "+m" (*(ptr)) \
31a8394e JF	49	: : "memory", "cc"); \
	50	break; \
	51	case __X86_CASE_W: \
	52	asm volatile (lock #op "w %w0, %1\n" \
	53	: "+r" (__ret), "+m" (*(ptr)) \
	54	: : "memory", "cc"); \
	55	break; \
	56	case __X86_CASE_L: \
	57	asm volatile (lock #op "l %0, %1\n" \
	58	: "+r" (__ret), "+m" (*(ptr)) \
	59	: : "memory", "cc"); \
	60	break; \
	61	case __X86_CASE_Q: \
	62	asm volatile (lock #op "q %q0, %1\n" \
	63	: "+r" (__ret), "+m" (*(ptr)) \
	64	: : "memory", "cc"); \
	65	break; \
	66	default: \
	67	__ ## op ## _wrong_size(); \
	68	} \
	69	__ret; \
	70	})
	71
e9826380 JF	72	/*
	73	* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
	74	* Since this is generally used to protect other memory information, we
	75	* use "asm volatile" and "memory" clobbers to prevent gcc from moving
	76	* information around.
	77	*/
31a8394e	78	#define xchg(ptr, v) __xchg_op((ptr), (v), xchg, "")
e9826380 JF	79
	80	/*
	81	* Atomic compare and exchange. Compare OLD with MEM, if identical,
	82	* store NEW in MEM. Return the initial value in MEM. Success is
	83	* indicated by comparing RETURN with OLD.
	84	*/
	85	#define __raw_cmpxchg(ptr, old, new, size, lock) \
	86	({ \
	87	__typeof__(*(ptr)) __ret; \
	88	__typeof__(*(ptr)) __old = (old); \
	89	__typeof__(*(ptr)) __new = (new); \
	90	switch (size) { \
	91	case __X86_CASE_B: \
	92	{ \
	93	volatile u8 __ptr = (volatile u8 )(ptr); \
	94	asm volatile(lock "cmpxchgb %2,%1" \
	95	: "=a" (__ret), "+m" (*__ptr) \
	96	: "q" (__new), "0" (__old) \
	97	: "memory"); \
	98	break; \
	99	} \
	100	case __X86_CASE_W: \
	101	{ \
	102	volatile u16 __ptr = (volatile u16 )(ptr); \
	103	asm volatile(lock "cmpxchgw %2,%1" \
	104	: "=a" (__ret), "+m" (*__ptr) \
	105	: "r" (__new), "0" (__old) \
	106	: "memory"); \
	107	break; \
	108	} \
	109	case __X86_CASE_L: \
	110	{ \
	111	volatile u32 __ptr = (volatile u32 )(ptr); \
	112	asm volatile(lock "cmpxchgl %2,%1" \
	113	: "=a" (__ret), "+m" (*__ptr) \
	114	: "r" (__new), "0" (__old) \
	115	: "memory"); \
	116	break; \
	117	} \
	118	case __X86_CASE_Q: \
	119	{ \
	120	volatile u64 __ptr = (volatile u64 )(ptr); \
	121	asm volatile(lock "cmpxchgq %2,%1" \
	122	: "=a" (__ret), "+m" (*__ptr) \
	123	: "r" (__new), "0" (__old) \
	124	: "memory"); \
	125	break; \
	126	} \
	127	default: \
	128	__cmpxchg_wrong_size(); \
	129	} \
	130	__ret; \
	131	})
	132
	133	#define __cmpxchg(ptr, old, new, size) \
	134	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
	135
	136	#define __sync_cmpxchg(ptr, old, new, size) \
	137	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
	138
	139	#define __cmpxchg_local(ptr, old, new, size) \
	140	__raw_cmpxchg((ptr), (old), (new), (size), "")
	141
96a388de	142	#ifdef CONFIG_X86_32
a1ce3928	143	# include <asm/cmpxchg_32.h>
96a388de	144	#else
a1ce3928	145	# include <asm/cmpxchg_64.h>
96a388de	146	#endif
e9826380	147
e9826380	148	#define cmpxchg(ptr, old, new) \
fc395b92	149	__cmpxchg(ptr, old, new, sizeof(*(ptr)))
e9826380 JF	150
e9826380 JF	151	#define sync_cmpxchg(ptr, old, new) \
fc395b92	152	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))
e9826380 JF	153
e9826380 JF	154	#define cmpxchg_local(ptr, old, new) \
fc395b92	155	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))
e9826380	156
433b3520 JF	157	/*
	158	* xadd() adds "inc" to "*ptr" and atomically returns the previous
	159	* value of "*ptr".
	160	*
	161	* xadd() is locked when multiple CPUs are online
	162	* xadd_sync() is always locked
	163	* xadd_local() is never locked
	164	*/
31a8394e	165	#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
433b3520 JF	166	#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
	167	#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")
	168	#define xadd_local(ptr, inc) __xadd((ptr), (inc), "")
	169
3d94ae0c JF	170	#define __add(ptr, inc, lock) \
	171	({ \
	172	__typeof__ (*(ptr)) __ret = (inc); \
	173	switch (sizeof(*(ptr))) { \
	174	case __X86_CASE_B: \
	175	asm volatile (lock "addb %b1, %0\n" \
8c91c532	176	: "+m" (*(ptr)) : "qi" (inc) \
3d94ae0c JF	177	: "memory", "cc"); \
	178	break; \
	179	case __X86_CASE_W: \
	180	asm volatile (lock "addw %w1, %0\n" \
	181	: "+m" (*(ptr)) : "ri" (inc) \
	182	: "memory", "cc"); \
	183	break; \
	184	case __X86_CASE_L: \
	185	asm volatile (lock "addl %1, %0\n" \
	186	: "+m" (*(ptr)) : "ri" (inc) \
	187	: "memory", "cc"); \
	188	break; \
	189	case __X86_CASE_Q: \
	190	asm volatile (lock "addq %1, %0\n" \
	191	: "+m" (*(ptr)) : "ri" (inc) \
	192	: "memory", "cc"); \
	193	break; \
	194	default: \
	195	__add_wrong_size(); \
	196	} \
	197	__ret; \
	198	})
	199
	200	/*
	201	* add_() adds "inc" to "ptr"
	202	*
	203	* __add() takes a lock prefix
	204	* add_smp() is locked when multiple CPUs are online
	205	* add_sync() is always locked
	206	*/
	207	#define add_smp(ptr, inc) __add((ptr), (inc), LOCK_PREFIX)
	208	#define add_sync(ptr, inc) __add((ptr), (inc), "lock; ")
	209
cdcd6298 JB	210	#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
	211	({ \
	212	bool __ret; \
	213	__typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \
	214	__typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \
	215	BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
	216	BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
	217	VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \
	218	VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \
	219	asm volatile(pfx "cmpxchg%c4b %2; sete %0" \
	220	: "=a" (__ret), "+d" (__old2), \
	221	"+m" ((p1)), "+m" ((p2)) \
	222	: "i" (2 * sizeof(long)), "a" (__old1), \
	223	"b" (__new1), "c" (__new2)); \
	224	__ret; \
	225	})
	226
	227	#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
	228	__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)
	229
	230	#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
	231	__cmpxchg_double(, p1, p2, o1, o2, n1, n2)
	232
e9826380	233	#endif /* ASM_X86_CMPXCHG_H */