[deliverable/linux.git] / arch / x86 / include / asm / cmpxchg_32.h

#ifndef _ASM_X86_CMPXCHG_32_H
#define _ASM_X86_CMPXCHG_32_H

#include <linux/bitops.h> /* for LOCK_PREFIX */

/*
 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
 *       you need to test for the feature in boot_cpu_data.
 */

extern void __xchg_wrong_size(void);

/*
 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
 * Since this is generally used to protect other memory information, we
 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
 * information around.
 */
#define __xchg(x, ptr, size)						\
({									\
	__typeof(*(ptr)) __x = (x);					\
	switch (size) {							\
	case 1:								\
	{								\
		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
		asm volatile("xchgb %0,%1"				\
			     : "=q" (__x), "+m" (*__ptr)		\
			     : "0" (__x)				\
			     : "memory");				\
		break;							\
	}								\
	case 2:								\
	{								\
		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
		asm volatile("xchgw %0,%1"				\
			     : "=r" (__x), "+m" (*__ptr)		\
			     : "0" (__x)				\
			     : "memory");				\
		break;							\
	}								\
	case 4:								\
	{								\
		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
		asm volatile("xchgl %0,%1"				\
			     : "=r" (__x), "+m" (*__ptr)		\
			     : "0" (__x)				\
			     : "memory");				\
		break;							\
	}								\
	default:							\
		__xchg_wrong_size();					\
	}								\
	__x;								\
})

#define xchg(ptr, v)							\
	__xchg((v), (ptr), sizeof(*ptr))

/*
 * CMPXCHG8B only writes to the target if we had the previous
 * value in registers, otherwise it acts as a read and gives us the
 * "new previous" value.  That is why there is a loop.  Preloading
 * EDX:EAX is a performance optimization: in the common case it means
 * we need only one locked operation.
 *
 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
 * least an FPU save and/or %cr0.ts manipulation.
 *
 * cmpxchg8b must be used with the lock prefix here to allow the
 * instruction to be executed atomically.  We need to have the reader
 * side to see the coherent 64bit value.
 */
static inline void set_64bit(volatile u64 *ptr, u64 value)
{
	u32 low  = value;
	u32 high = value >> 32;
	u64 prev = *ptr;

	asm volatile("\n1:\t"
		     LOCK_PREFIX "cmpxchg8b %0\n\t"
		     "jnz 1b"
		     : "=m" (*ptr), "+A" (prev)
		     : "b" (low), "c" (high)
		     : "memory");
}

extern void __cmpxchg_wrong_size(void);

/*
 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 * store NEW in MEM.  Return the initial value in MEM.  Success is
 * indicated by comparing RETURN with OLD.
 */
#define __raw_cmpxchg(ptr, old, new, size, lock)			\
({									\
	__typeof__(*(ptr)) __ret;					\
	__typeof__(*(ptr)) __old = (old);				\
	__typeof__(*(ptr)) __new = (new);				\
	switch (size) {							\
	case 1:								\
	{								\
		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
		asm volatile(lock "cmpxchgb %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "q" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case 2:								\
	{								\
		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
		asm volatile(lock "cmpxchgw %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case 4:								\
	{								\
		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
		asm volatile(lock "cmpxchgl %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	default:							\
		__cmpxchg_wrong_size();					\
	}								\
	__ret;								\
})

#define __cmpxchg(ptr, old, new, size)					\
	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)

#define __sync_cmpxchg(ptr, old, new, size)				\
	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")

#define __cmpxchg_local(ptr, old, new, size)				\
	__raw_cmpxchg((ptr), (old), (new), (size), "")

#ifdef CONFIG_X86_CMPXCHG
#define __HAVE_ARCH_CMPXCHG 1

#define cmpxchg(ptr, old, new)						\
	__cmpxchg((ptr), (old), (new), sizeof(*ptr))

#define sync_cmpxchg(ptr, old, new)					\
	__sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))

#define cmpxchg_local(ptr, old, new)					\
	__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
#endif

#ifdef CONFIG_X86_CMPXCHG64
#define cmpxchg64(ptr, o, n)						\
	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
					 (unsigned long long)(n)))
#define cmpxchg64_local(ptr, o, n)					\
	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
					       (unsigned long long)(n)))
#endif

static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
{
	u64 prev;
	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
		     : "=A" (prev),
		       "+m" (*ptr)
		     : "b" ((u32)new),
		       "c" ((u32)(new >> 32)),
		       "0" (old)
		     : "memory");
	return prev;
}

static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
{
	u64 prev;
	asm volatile("cmpxchg8b %1"
		     : "=A" (prev),
		       "+m" (*ptr)
		     : "b" ((u32)new),
		       "c" ((u32)(new >> 32)),
		       "0" (old)
		     : "memory");
	return prev;
}

#ifndef CONFIG_X86_CMPXCHG
/*
 * Building a kernel capable running on 80386. It may be necessary to
 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
 * a function for each of the sizes we support.
 */

extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);

static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
					unsigned long new, int size)
{
	switch (size) {
	case 1:
		return cmpxchg_386_u8(ptr, old, new);
	case 2:
		return cmpxchg_386_u16(ptr, old, new);
	case 4:
		return cmpxchg_386_u32(ptr, old, new);
	}
	return old;
}

#define cmpxchg(ptr, o, n)						\
({									\
	__typeof__(*(ptr)) __ret;					\
	if (likely(boot_cpu_data.x86 > 3))				\
		__ret = (__typeof__(*(ptr)))__cmpxchg((ptr),		\
				(unsigned long)(o), (unsigned long)(n),	\
				sizeof(*(ptr)));			\
	else								\
		__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),		\
				(unsigned long)(o), (unsigned long)(n),	\
				sizeof(*(ptr)));			\
	__ret;								\
})
#define cmpxchg_local(ptr, o, n)					\
({									\
	__typeof__(*(ptr)) __ret;					\
	if (likely(boot_cpu_data.x86 > 3))				\
		__ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr),	\
				(unsigned long)(o), (unsigned long)(n),	\
				sizeof(*(ptr)));			\
	else								\
		__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),		\
				(unsigned long)(o), (unsigned long)(n),	\
				sizeof(*(ptr)));			\
	__ret;								\
})
#endif

#ifndef CONFIG_X86_CMPXCHG64
/*
 * Building a kernel capable running on 80386 and 80486. It may be necessary
 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
 */

#define cmpxchg64(ptr, o, n)					\
({								\
	__typeof__(*(ptr)) __ret;				\
	__typeof__(*(ptr)) __old = (o);				\
	__typeof__(*(ptr)) __new = (n);				\
	alternative_io(LOCK_PREFIX_HERE				\
			"call cmpxchg8b_emu",			\
			"lock; cmpxchg8b (%%esi)" ,		\
		       X86_FEATURE_CX8,				\
		       "=A" (__ret),				\
		       "S" ((ptr)), "0" (__old),		\
		       "b" ((unsigned int)__new),		\
		       "c" ((unsigned int)(__new>>32))		\
		       : "memory");				\
	__ret; })


#define cmpxchg64_local(ptr, o, n)				\
({								\
	__typeof__(*(ptr)) __ret;				\
	__typeof__(*(ptr)) __old = (o);				\
	__typeof__(*(ptr)) __new = (n);				\
	alternative_io("call cmpxchg8b_emu",			\
		       "cmpxchg8b (%%esi)" ,			\
		       X86_FEATURE_CX8,				\
		       "=A" (__ret),				\
		       "S" ((ptr)), "0" (__old),		\
		       "b" ((unsigned int)__new),		\
		       "c" ((unsigned int)(__new>>32))		\
		       : "memory");				\
	__ret; })

#endif

#endif /* _ASM_X86_CMPXCHG_32_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_CMPXCHG_32_H
1965aae3 PA	2	#define _ASM_X86_CMPXCHG_32_H
a436ed9c JD	3
	4	#include <linux/bitops.h> /* for LOCK_PREFIX */
	5
2d9ce177 AK	6	/*
	7	* Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
	8	* you need to test for the feature in boot_cpu_data.
	9	*/
	10
f3834b9e PZ	11	extern void __xchg_wrong_size(void);
	12
	13	/*
4532b305 PA	14	* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
	15	* Since this is generally used to protect other memory information, we
	16	* use "asm volatile" and "memory" clobbers to prevent gcc from moving
	17	* information around.
f3834b9e	18	*/
f3834b9e PZ	19	#define __xchg(x, ptr, size) \
	20	({ \
	21	__typeof(*(ptr)) __x = (x); \
	22	switch (size) { \
	23	case 1: \
4532b305 PA	24	{ \
	25	volatile u8 __ptr = (volatile u8 )(ptr); \
	26	asm volatile("xchgb %0,%1" \
	27	: "=q" (__x), "+m" (*__ptr) \
113fc5a6	28	: "0" (__x) \
f3834b9e PZ	29	: "memory"); \
f3834b9e PZ	30	break; \
4532b305	31	} \
f3834b9e	32	case 2: \
4532b305 PA	33	{ \
	34	volatile u16 __ptr = (volatile u16 )(ptr); \
	35	asm volatile("xchgw %0,%1" \
	36	: "=r" (__x), "+m" (*__ptr) \
113fc5a6	37	: "0" (__x) \
f3834b9e PZ	38	: "memory"); \
f3834b9e PZ	39	break; \
4532b305	40	} \
f3834b9e	41	case 4: \
4532b305 PA	42	{ \
4532b305 PA	43	volatile u32 __ptr = (volatile u32 )(ptr); \
f3834b9e	44	asm volatile("xchgl %0,%1" \
4532b305	45	: "=r" (__x), "+m" (*__ptr) \
113fc5a6	46	: "0" (__x) \
f3834b9e PZ	47	: "memory"); \
f3834b9e PZ	48	break; \
4532b305	49	} \
f3834b9e PZ	50	default: \
	51	__xchg_wrong_size(); \
	52	} \
	53	__x; \
	54	})
	55
	56	#define xchg(ptr, v) \
	57	__xchg((v), (ptr), sizeof(*ptr))
	58
a436ed9c	59	/*
69309a05 PA	60	* CMPXCHG8B only writes to the target if we had the previous
	61	* value in registers, otherwise it acts as a read and gives us the
	62	* "new previous" value. That is why there is a loop. Preloading
	63	* EDX:EAX is a performance optimization: in the common case it means
	64	* we need only one locked operation.
a436ed9c	65	*
69309a05 PA	66	* A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
	67	* least an FPU save and/or %cr0.ts manipulation.
	68	*
	69	* cmpxchg8b must be used with the lock prefix here to allow the
	70	* instruction to be executed atomically. We need to have the reader
	71	* side to see the coherent 64bit value.
a436ed9c	72	*/
69309a05	73	static inline void set_64bit(volatile u64 *ptr, u64 value)
a436ed9c	74	{
69309a05 PA	75	u32 low = value;
	76	u32 high = value >> 32;
	77	u64 prev = *ptr;
	78
8121019c	79	asm volatile("\n1:\t"
69309a05	80	LOCK_PREFIX "cmpxchg8b %0\n\t"
8121019c	81	"jnz 1b"
69309a05 PA	82	: "=m" (*ptr), "+A" (prev)
	83	: "b" (low), "c" (high)
	84	: "memory");
a436ed9c	85	}
a436ed9c	86
f3834b9e	87	extern void __cmpxchg_wrong_size(void);
a436ed9c JD	88
	89	/*
	90	* Atomic compare and exchange. Compare OLD with MEM, if identical,
	91	* store NEW in MEM. Return the initial value in MEM. Success is
	92	* indicated by comparing RETURN with OLD.
	93	*/
f3834b9e PZ	94	#define __raw_cmpxchg(ptr, old, new, size, lock) \
	95	({ \
	96	__typeof__(*(ptr)) __ret; \
	97	__typeof__(*(ptr)) __old = (old); \
	98	__typeof__(*(ptr)) __new = (new); \
	99	switch (size) { \
	100	case 1: \
4532b305 PA	101	{ \
	102	volatile u8 __ptr = (volatile u8 )(ptr); \
	103	asm volatile(lock "cmpxchgb %2,%1" \
	104	: "=a" (__ret), "+m" (*__ptr) \
113fc5a6	105	: "q" (__new), "0" (__old) \
f3834b9e PZ	106	: "memory"); \
f3834b9e PZ	107	break; \
4532b305	108	} \
f3834b9e	109	case 2: \
4532b305 PA	110	{ \
	111	volatile u16 __ptr = (volatile u16 )(ptr); \
	112	asm volatile(lock "cmpxchgw %2,%1" \
	113	: "=a" (__ret), "+m" (*__ptr) \
113fc5a6	114	: "r" (__new), "0" (__old) \
f3834b9e PZ	115	: "memory"); \
f3834b9e PZ	116	break; \
4532b305	117	} \
f3834b9e	118	case 4: \
4532b305 PA	119	{ \
4532b305 PA	120	volatile u32 __ptr = (volatile u32 )(ptr); \
113fc5a6	121	asm volatile(lock "cmpxchgl %2,%1" \
4532b305	122	: "=a" (__ret), "+m" (*__ptr) \
113fc5a6	123	: "r" (__new), "0" (__old) \
f3834b9e PZ	124	: "memory"); \
f3834b9e PZ	125	break; \
4532b305	126	} \
f3834b9e PZ	127	default: \
	128	__cmpxchg_wrong_size(); \
	129	} \
	130	__ret; \
	131	})
	132
	133	#define __cmpxchg(ptr, old, new, size) \
	134	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
	135
	136	#define __sync_cmpxchg(ptr, old, new, size) \
	137	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
	138
	139	#define __cmpxchg_local(ptr, old, new, size) \
	140	__raw_cmpxchg((ptr), (old), (new), (size), "")
a436ed9c JD	141
	142	#ifdef CONFIG_X86_CMPXCHG
	143	#define __HAVE_ARCH_CMPXCHG 1
f3834b9e PZ	144
	145	#define cmpxchg(ptr, old, new) \
	146	__cmpxchg((ptr), (old), (new), sizeof(*ptr))
	147
	148	#define sync_cmpxchg(ptr, old, new) \
	149	__sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
	150
	151	#define cmpxchg_local(ptr, old, new) \
	152	__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
2c0b8a75 MD	153	#endif
	154
	155	#ifdef CONFIG_X86_CMPXCHG64
8121019c JP	156	#define cmpxchg64(ptr, o, n) \
	157	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
	158	(unsigned long long)(n)))
	159	#define cmpxchg64_local(ptr, o, n) \
	160	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
	161	(unsigned long long)(n)))
a436ed9c JD	162	#endif
a436ed9c JD	163
4532b305	164	static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
2c0b8a75	165	{
4532b305	166	u64 prev;
113fc5a6 PA	167	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
113fc5a6 PA	168	: "=A" (prev),
4532b305 PA	169	"+m" (*ptr)
	170	: "b" ((u32)new),
	171	"c" ((u32)(new >> 32)),
113fc5a6	172	"0" (old)
8121019c	173	: "memory");
2c0b8a75 MD	174	return prev;
	175	}
	176
4532b305	177	static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
2c0b8a75	178	{
4532b305	179	u64 prev;
113fc5a6 PA	180	asm volatile("cmpxchg8b %1"
113fc5a6 PA	181	: "=A" (prev),
4532b305 PA	182	"+m" (*ptr)
	183	: "b" ((u32)new),
	184	"c" ((u32)(new >> 32)),
113fc5a6	185	"0" (old)
8121019c	186	: "memory");
2c0b8a75 MD	187	return prev;
	188	}
	189
a436ed9c JD	190	#ifndef CONFIG_X86_CMPXCHG
	191	/*
	192	* Building a kernel capable running on 80386. It may be necessary to
	193	* simulate the cmpxchg on the 80386 CPU. For that purpose we define
	194	* a function for each of the sizes we support.
	195	*/
	196
	197	extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
	198	extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
	199	extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
	200
	201	static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
8121019c	202	unsigned long new, int size)
a436ed9c JD	203	{
	204	switch (size) {
	205	case 1:
	206	return cmpxchg_386_u8(ptr, old, new);
	207	case 2:
	208	return cmpxchg_386_u16(ptr, old, new);
	209	case 4:
	210	return cmpxchg_386_u32(ptr, old, new);
	211	}
	212	return old;
	213	}
	214
2c0b8a75	215	#define cmpxchg(ptr, o, n) \
a436ed9c JD	216	({ \
	217	__typeof__(*(ptr)) __ret; \
	218	if (likely(boot_cpu_data.x86 > 3)) \
3078b79d MD	219	__ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \
	220	(unsigned long)(o), (unsigned long)(n), \
	221	sizeof(*(ptr))); \
a436ed9c	222	else \
3078b79d MD	223	__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
	224	(unsigned long)(o), (unsigned long)(n), \
	225	sizeof(*(ptr))); \
a436ed9c JD	226	__ret; \
a436ed9c JD	227	})
2c0b8a75	228	#define cmpxchg_local(ptr, o, n) \
a436ed9c JD	229	({ \
	230	__typeof__(*(ptr)) __ret; \
	231	if (likely(boot_cpu_data.x86 > 3)) \
3078b79d MD	232	__ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \
	233	(unsigned long)(o), (unsigned long)(n), \
	234	sizeof(*(ptr))); \
a436ed9c	235	else \
3078b79d MD	236	__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
	237	(unsigned long)(o), (unsigned long)(n), \
	238	sizeof(*(ptr))); \
a436ed9c JD	239	__ret; \
	240	})
	241	#endif
	242
2c0b8a75 MD	243	#ifndef CONFIG_X86_CMPXCHG64
	244	/*
	245	* Building a kernel capable running on 80386 and 80486. It may be necessary
	246	* to simulate the cmpxchg8b on the 80386 and 80486 CPU.
	247	*/
a436ed9c	248
79e1dd05 AV	249	#define cmpxchg64(ptr, o, n) \
	250	({ \
	251	__typeof__(*(ptr)) __ret; \
	252	__typeof__(*(ptr)) __old = (o); \
	253	__typeof__(*(ptr)) __new = (n); \
9c76b384 LB	254	alternative_io(LOCK_PREFIX_HERE \
9c76b384 LB	255	"call cmpxchg8b_emu", \
79e1dd05 AV	256	"lock; cmpxchg8b (%%esi)" , \
	257	X86_FEATURE_CX8, \
	258	"=A" (__ret), \
	259	"S" ((ptr)), "0" (__old), \
	260	"b" ((unsigned int)__new), \
	261	"c" ((unsigned int)(__new>>32)) \
	262	: "memory"); \
	263	__ret; })
	264
	265
a378d933 PA	266	#define cmpxchg64_local(ptr, o, n) \
	267	({ \
	268	__typeof__(*(ptr)) __ret; \
	269	__typeof__(*(ptr)) __old = (o); \
	270	__typeof__(*(ptr)) __new = (n); \
	271	alternative_io("call cmpxchg8b_emu", \
	272	"cmpxchg8b (%%esi)" , \
	273	X86_FEATURE_CX8, \
	274	"=A" (__ret), \
	275	"S" ((ptr)), "0" (__old), \
	276	"b" ((unsigned int)__new), \
	277	"c" ((unsigned int)(__new>>32)) \
	278	: "memory"); \
	279	__ret; })
2c0b8a75 MD	280
2c0b8a75 MD	281	#endif
a436ed9c	282
1965aae3	283	#endif /* _ASM_X86_CMPXCHG_32_H */