[deliverable/linux.git] / arch / s390 / include / asm / atomic.h

/*
 * Copyright IBM Corp. 1999, 2009
 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
 *	      Denis Joseph Barrow,
 *	      Arnd Bergmann <arndb@de.ibm.com>,
 *
 * Atomic operations that C can't guarantee us.
 * Useful for resource counting etc.
 * s390 uses 'Compare And Swap' for atomicity in SMP environment.
 *
 */

#ifndef __ARCH_S390_ATOMIC__
#define __ARCH_S390_ATOMIC__

#include <linux/compiler.h>
#include <linux/types.h>
#include <asm/barrier.h>
#include <asm/cmpxchg.h>

#define ATOMIC_INIT(i)  { (i) }

#define __ATOMIC_NO_BARRIER	"\n"

#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES

#define __ATOMIC_OR	"lao"
#define __ATOMIC_AND	"lan"
#define __ATOMIC_ADD	"laa"
#define __ATOMIC_BARRIER "bcr	14,0\n"

#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\
({									\
	int old_val;							\
									\
	typecheck(atomic_t *, ptr);					\
	asm volatile(							\
		__barrier						\
		op_string "	%0,%2,%1\n"				\
		__barrier						\
		: "=d" (old_val), "+Q" ((ptr)->counter)			\
		: "d" (op_val)						\
		: "cc", "memory");					\
	old_val;							\
})

#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */

#define __ATOMIC_OR	"or"
#define __ATOMIC_AND	"nr"
#define __ATOMIC_ADD	"ar"
#define __ATOMIC_BARRIER "\n"

#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\
({									\
	int old_val, new_val;						\
									\
	typecheck(atomic_t *, ptr);					\
	asm volatile(							\
		"	l	%0,%2\n"				\
		"0:	lr	%1,%0\n"				\
		op_string "	%1,%3\n"				\
		"	cs	%0,%1,%2\n"				\
		"	jl	0b"					\
		: "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
		: "d" (op_val)						\
		: "cc", "memory");					\
	old_val;							\
})

#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */

static inline int atomic_read(const atomic_t *v)
{
	int c;

	asm volatile(
		"	l	%0,%1\n"
		: "=d" (c) : "Q" (v->counter));
	return c;
}

static inline void atomic_set(atomic_t *v, int i)
{
	asm volatile(
		"	st	%1,%0\n"
		: "=Q" (v->counter) : "d" (i));
}

static inline int atomic_add_return(int i, atomic_t *v)
{
	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;
}

static inline void atomic_add(int i, atomic_t *v)
{
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
		asm volatile(
			"asi	%0,%1\n"
			: "+Q" (v->counter)
			: "i" (i)
			: "cc", "memory");
		return;
	}
#endif
	__ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);
}

#define atomic_add_negative(_i, _v)	(atomic_add_return(_i, _v) < 0)
#define atomic_inc(_v)			atomic_add(1, _v)
#define atomic_inc_return(_v)		atomic_add_return(1, _v)
#define atomic_inc_and_test(_v)		(atomic_add_return(1, _v) == 0)
#define atomic_sub(_i, _v)		atomic_add(-(int)(_i), _v)
#define atomic_sub_return(_i, _v)	atomic_add_return(-(int)(_i), _v)
#define atomic_sub_and_test(_i, _v)	(atomic_sub_return(_i, _v) == 0)
#define atomic_dec(_v)			atomic_sub(1, _v)
#define atomic_dec_return(_v)		atomic_sub_return(1, _v)
#define atomic_dec_and_test(_v)		(atomic_sub_return(1, _v) == 0)

static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
	__ATOMIC_LOOP(v, ~mask, __ATOMIC_AND, __ATOMIC_NO_BARRIER);
}

static inline void atomic_set_mask(unsigned int mask, atomic_t *v)
{
	__ATOMIC_LOOP(v, mask, __ATOMIC_OR, __ATOMIC_NO_BARRIER);
}

#define atomic_xchg(v, new) (xchg(&((v)->counter), new))

static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
{
	asm volatile(
		"	cs	%0,%2,%1"
		: "+d" (old), "+Q" (v->counter)
		: "d" (new)
		: "cc", "memory");
	return old;
}

static inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
	int c, old;
	c = atomic_read(v);
	for (;;) {
		if (unlikely(c == u))
			break;
		old = atomic_cmpxchg(v, c, c + a);
		if (likely(old == c))
			break;
		c = old;
	}
	return c;
}


#undef __ATOMIC_LOOP

#define ATOMIC64_INIT(i)  { (i) }

#ifdef CONFIG_64BIT

#define __ATOMIC64_NO_BARRIER	"\n"

#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES

#define __ATOMIC64_OR	"laog"
#define __ATOMIC64_AND	"lang"
#define __ATOMIC64_ADD	"laag"
#define __ATOMIC64_BARRIER "bcr	14,0\n"

#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\
({									\
	long long old_val;						\
									\
	typecheck(atomic64_t *, ptr);					\
	asm volatile(							\
		__barrier						\
		op_string "	%0,%2,%1\n"				\
		__barrier						\
		: "=d" (old_val), "+Q" ((ptr)->counter)			\
		: "d" (op_val)						\
		: "cc", "memory");					\
	old_val;							\
})

#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */

#define __ATOMIC64_OR	"ogr"
#define __ATOMIC64_AND	"ngr"
#define __ATOMIC64_ADD	"agr"
#define __ATOMIC64_BARRIER "\n"

#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\
({									\
	long long old_val, new_val;					\
									\
	typecheck(atomic64_t *, ptr);					\
	asm volatile(							\
		"	lg	%0,%2\n"				\
		"0:	lgr	%1,%0\n"				\
		op_string "	%1,%3\n"				\
		"	csg	%0,%1,%2\n"				\
		"	jl	0b"					\
		: "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
		: "d" (op_val)						\
		: "cc", "memory");					\
	old_val;							\
})

#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */

static inline long long atomic64_read(const atomic64_t *v)
{
	long long c;

	asm volatile(
		"	lg	%0,%1\n"
		: "=d" (c) : "Q" (v->counter));
	return c;
}

static inline void atomic64_set(atomic64_t *v, long long i)
{
	asm volatile(
		"	stg	%1,%0\n"
		: "=Q" (v->counter) : "d" (i));
}

static inline long long atomic64_add_return(long long i, atomic64_t *v)
{
	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i;
}

static inline void atomic64_add(long long i, atomic64_t *v)
{
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
		asm volatile(
			"agsi	%0,%1\n"
			: "+Q" (v->counter)
			: "i" (i)
			: "cc", "memory");
		return;
	}
#endif
	__ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER);
}

static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)
{
	__ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND, __ATOMIC64_NO_BARRIER);
}

static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)
{
	__ATOMIC64_LOOP(v, mask, __ATOMIC64_OR, __ATOMIC64_NO_BARRIER);
}

#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))

static inline long long atomic64_cmpxchg(atomic64_t *v,
					     long long old, long long new)
{
	asm volatile(
		"	csg	%0,%2,%1"
		: "+d" (old), "+Q" (v->counter)
		: "d" (new)
		: "cc", "memory");
	return old;
}

#undef __ATOMIC64_LOOP

#else /* CONFIG_64BIT */

typedef struct {
	long long counter;
} atomic64_t;

static inline long long atomic64_read(const atomic64_t *v)
{
	register_pair rp;

	asm volatile(
		"	lm	%0,%N0,%1"
		: "=&d" (rp) : "Q" (v->counter)	);
	return rp.pair;
}

static inline void atomic64_set(atomic64_t *v, long long i)
{
	register_pair rp = {.pair = i};

	asm volatile(
		"	stm	%1,%N1,%0"
		: "=Q" (v->counter) : "d" (rp) );
}

static inline long long atomic64_xchg(atomic64_t *v, long long new)
{
	register_pair rp_new = {.pair = new};
	register_pair rp_old;

	asm volatile(
		"	lm	%0,%N0,%1\n"
		"0:	cds	%0,%2,%1\n"
		"	jl	0b\n"
		: "=&d" (rp_old), "+Q" (v->counter)
		: "d" (rp_new)
		: "cc");
	return rp_old.pair;
}

static inline long long atomic64_cmpxchg(atomic64_t *v,
					 long long old, long long new)
{
	register_pair rp_old = {.pair = old};
	register_pair rp_new = {.pair = new};

	asm volatile(
		"	cds	%0,%2,%1"
		: "+&d" (rp_old), "+Q" (v->counter)
		: "d" (rp_new)
		: "cc");
	return rp_old.pair;
}


static inline long long atomic64_add_return(long long i, atomic64_t *v)
{
	long long old, new;

	do {
		old = atomic64_read(v);
		new = old + i;
	} while (atomic64_cmpxchg(v, old, new) != old);
	return new;
}

static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v)
{
	long long old, new;

	do {
		old = atomic64_read(v);
		new = old | mask;
	} while (atomic64_cmpxchg(v, old, new) != old);
}

static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v)
{
	long long old, new;

	do {
		old = atomic64_read(v);
		new = old & mask;
	} while (atomic64_cmpxchg(v, old, new) != old);
}

static inline void atomic64_add(long long i, atomic64_t *v)
{
	atomic64_add_return(i, v);
}

#endif /* CONFIG_64BIT */

static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)
{
	long long c, old;

	c = atomic64_read(v);
	for (;;) {
		if (unlikely(c == u))
			break;
		old = atomic64_cmpxchg(v, c, c + i);
		if (likely(old == c))
			break;
		c = old;
	}
	return c != u;
}

static inline long long atomic64_dec_if_positive(atomic64_t *v)
{
	long long c, old, dec;

	c = atomic64_read(v);
	for (;;) {
		dec = c - 1;
		if (unlikely(dec < 0))
			break;
		old = atomic64_cmpxchg((v), c, dec);
		if (likely(old == c))
			break;
		c = old;
	}
	return dec;
}

#define atomic64_add_negative(_i, _v)	(atomic64_add_return(_i, _v) < 0)
#define atomic64_inc(_v)		atomic64_add(1, _v)
#define atomic64_inc_return(_v)		atomic64_add_return(1, _v)
#define atomic64_inc_and_test(_v)	(atomic64_add_return(1, _v) == 0)
#define atomic64_sub_return(_i, _v)	atomic64_add_return(-(long long)(_i), _v)
#define atomic64_sub(_i, _v)		atomic64_add(-(long long)(_i), _v)
#define atomic64_sub_and_test(_i, _v)	(atomic64_sub_return(_i, _v) == 0)
#define atomic64_dec(_v)		atomic64_sub(1, _v)
#define atomic64_dec_return(_v)		atomic64_sub_return(1, _v)
#define atomic64_dec_and_test(_v)	(atomic64_sub_return(1, _v) == 0)
#define atomic64_inc_not_zero(v)	atomic64_add_unless((v), 1, 0)

#define smp_mb__before_atomic_dec()	smp_mb()
#define smp_mb__after_atomic_dec()	smp_mb()
#define smp_mb__before_atomic_inc()	smp_mb()
#define smp_mb__after_atomic_inc()	smp_mb()

#endif /* __ARCH_S390_ATOMIC__  */
Commit	Line	Data
1da177e4	1	/*
a53c8fab	2	* Copyright IBM Corp. 1999, 2009
12751058 HC	3	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
	4	* Denis Joseph Barrow,
	5	* Arnd Bergmann <arndb@de.ibm.com>,
1da177e4	6	*
12751058 HC	7	* Atomic operations that C can't guarantee us.
12751058 HC	8	* Useful for resource counting etc.
25985edc	9	* s390 uses 'Compare And Swap' for atomicity in SMP environment.
1da177e4 LT	10	*
	11	*/
	12
a53c8fab HC	13	#ifndef __ARCH_S390_ATOMIC__
	14	#define __ARCH_S390_ATOMIC__
	15
12751058 HC	16	#include <linux/compiler.h>
12751058 HC	17	#include <linux/types.h>
0ccc8b7a	18	#include <asm/barrier.h>
a0616cde	19	#include <asm/cmpxchg.h>
1da177e4	20
1da177e4 LT	21	#define ATOMIC_INIT(i) { (i) }
1da177e4 LT	22
0ccc8b7a HC	23	#define __ATOMIC_NO_BARRIER "\n"
0ccc8b7a HC	24
75287430 HC	25	#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	26
	27	#define __ATOMIC_OR "lao"
	28	#define __ATOMIC_AND "lan"
	29	#define __ATOMIC_ADD "laa"
0ccc8b7a	30	#define __ATOMIC_BARRIER "bcr 14,0\n"
75287430	31
0ccc8b7a	32	#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \
75287430 HC	33	({ \
75287430 HC	34	int old_val; \
9a70a428 HC	35	\
9a70a428 HC	36	typecheck(atomic_t *, ptr); \
75287430	37	asm volatile( \
0ccc8b7a	38	__barrier \
75287430	39	op_string " %0,%2,%1\n" \
0ccc8b7a	40	__barrier \
9a70a428	41	: "=d" (old_val), "+Q" ((ptr)->counter) \
75287430 HC	42	: "d" (op_val) \
	43	: "cc", "memory"); \
	44	old_val; \
	45	})
	46
	47	#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
	48
	49	#define __ATOMIC_OR "or"
	50	#define __ATOMIC_AND "nr"
	51	#define __ATOMIC_ADD "ar"
0ccc8b7a	52	#define __ATOMIC_BARRIER "\n"
75287430	53
0ccc8b7a	54	#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \
75287430	55	({ \
39475179	56	int old_val, new_val; \
9a70a428 HC	57	\
9a70a428 HC	58	typecheck(atomic_t *, ptr); \
94c12cc7 MS	59	asm volatile( \
	60	" l %0,%2\n" \
	61	"0: lr %1,%0\n" \
	62	op_string " %1,%3\n" \
	63	" cs %0,%1,%2\n" \
	64	" jl 0b" \
9a70a428 HC	65	: "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
9a70a428 HC	66	: "d" (op_val) \
94c12cc7	67	: "cc", "memory"); \
75287430	68	old_val; \
1da177e4	69	})
94c12cc7	70
75287430 HC	71	#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
75287430 HC	72
c51b9621 HC	73	static inline int atomic_read(const atomic_t *v)
c51b9621 HC	74	{
7657e41a HC	75	int c;
	76
	77	asm volatile(
	78	" l %0,%1\n"
	79	: "=d" (c) : "Q" (v->counter));
	80	return c;
c51b9621 HC	81	}
	82
	83	static inline void atomic_set(atomic_t *v, int i)
	84	{
7657e41a HC	85	asm volatile(
	86	" st %1,%0\n"
	87	: "=Q" (v->counter) : "d" (i));
c51b9621	88	}
1da177e4	89
bfe3349b	90	static inline int atomic_add_return(int i, atomic_t *v)
1da177e4	91	{
0ccc8b7a	92	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;
1da177e4	93	}
75287430	94
5692e4d1 HC	95	static inline void atomic_add(int i, atomic_t *v)
	96	{
	97	#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	98	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
	99	asm volatile(
	100	"asi %0,%1\n"
	101	: "+Q" (v->counter)
	102	: "i" (i)
	103	: "cc", "memory");
0ccc8b7a	104	return;
5692e4d1	105	}
5692e4d1	106	#endif
0ccc8b7a	107	__ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);
5692e4d1 HC	108	}
5692e4d1 HC	109
973bd993	110	#define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0)
5692e4d1	111	#define atomic_inc(_v) atomic_add(1, _v)
973bd993 MS	112	#define atomic_inc_return(_v) atomic_add_return(1, _v)
973bd993 MS	113	#define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0)
5692e4d1	114	#define atomic_sub(_i, _v) atomic_add(-(int)(_i), _v)
86d51bc3	115	#define atomic_sub_return(_i, _v) atomic_add_return(-(int)(_i), _v)
973bd993	116	#define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0)
5692e4d1	117	#define atomic_dec(_v) atomic_sub(1, _v)
973bd993 MS	118	#define atomic_dec_return(_v) atomic_sub_return(1, _v)
973bd993 MS	119	#define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0)
1da177e4	120
72b7fb5f	121	static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
1da177e4	122	{
0ccc8b7a	123	__ATOMIC_LOOP(v, ~mask, __ATOMIC_AND, __ATOMIC_NO_BARRIER);
1da177e4	124	}
973bd993	125
72b7fb5f	126	static inline void atomic_set_mask(unsigned int mask, atomic_t *v)
1da177e4	127	{
0ccc8b7a	128	__ATOMIC_LOOP(v, mask, __ATOMIC_OR, __ATOMIC_NO_BARRIER);
1da177e4	129	}
973bd993	130
ffbf670f IM	131	#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
ffbf670f IM	132
bfe3349b	133	static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
973bd993	134	{
94c12cc7 MS	135	asm volatile(
94c12cc7 MS	136	" cs %0,%2,%1"
9a70a428 HC	137	: "+d" (old), "+Q" (v->counter)
9a70a428 HC	138	: "d" (new)
94c12cc7	139	: "cc", "memory");
973bd993 MS	140	return old;
	141	}
	142
f24219b4	143	static inline int __atomic_add_unless(atomic_t *v, int a, int u)
973bd993 MS	144	{
973bd993 MS	145	int c, old;
973bd993	146	c = atomic_read(v);
0b2fcfdb NP	147	for (;;) {
	148	if (unlikely(c == u))
	149	break;
	150	old = atomic_cmpxchg(v, c, c + a);
	151	if (likely(old == c))
	152	break;
973bd993	153	c = old;
0b2fcfdb	154	}
f24219b4	155	return c;
973bd993 MS	156	}
973bd993 MS	157
973bd993	158
75287430	159	#undef __ATOMIC_LOOP
1da177e4	160
1da177e4 LT	161	#define ATOMIC64_INIT(i) { (i) }
1da177e4 LT	162
12751058 HC	163	#ifdef CONFIG_64BIT
12751058 HC	164
0ccc8b7a HC	165	#define __ATOMIC64_NO_BARRIER "\n"
0ccc8b7a HC	166
75287430 HC	167	#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	168
	169	#define __ATOMIC64_OR "laog"
	170	#define __ATOMIC64_AND "lang"
	171	#define __ATOMIC64_ADD "laag"
0ccc8b7a	172	#define __ATOMIC64_BARRIER "bcr 14,0\n"
75287430	173
0ccc8b7a	174	#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \
75287430 HC	175	({ \
75287430 HC	176	long long old_val; \
9a70a428 HC	177	\
9a70a428 HC	178	typecheck(atomic64_t *, ptr); \
75287430	179	asm volatile( \
0ccc8b7a	180	__barrier \
75287430	181	op_string " %0,%2,%1\n" \
0ccc8b7a	182	__barrier \
9a70a428	183	: "=d" (old_val), "+Q" ((ptr)->counter) \
75287430 HC	184	: "d" (op_val) \
	185	: "cc", "memory"); \
	186	old_val; \
	187	})
	188
	189	#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
	190
	191	#define __ATOMIC64_OR "ogr"
	192	#define __ATOMIC64_AND "ngr"
	193	#define __ATOMIC64_ADD "agr"
0ccc8b7a	194	#define __ATOMIC64_BARRIER "\n"
75287430	195
0ccc8b7a	196	#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \
75287430	197	({ \
39475179	198	long long old_val, new_val; \
9a70a428 HC	199	\
9a70a428 HC	200	typecheck(atomic64_t *, ptr); \
94c12cc7 MS	201	asm volatile( \
	202	" lg %0,%2\n" \
	203	"0: lgr %1,%0\n" \
	204	op_string " %1,%3\n" \
	205	" csg %0,%1,%2\n" \
	206	" jl 0b" \
9a70a428 HC	207	: "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
9a70a428 HC	208	: "d" (op_val) \
bfe3349b	209	: "cc", "memory"); \
75287430	210	old_val; \
1da177e4	211	})
94c12cc7	212
75287430 HC	213	#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
75287430 HC	214
c51b9621 HC	215	static inline long long atomic64_read(const atomic64_t *v)
c51b9621 HC	216	{
7657e41a HC	217	long long c;
	218
	219	asm volatile(
	220	" lg %0,%1\n"
	221	: "=d" (c) : "Q" (v->counter));
	222	return c;
c51b9621 HC	223	}
	224
	225	static inline void atomic64_set(atomic64_t *v, long long i)
	226	{
7657e41a HC	227	asm volatile(
	228	" stg %1,%0\n"
	229	: "=Q" (v->counter) : "d" (i));
c51b9621	230	}
1da177e4	231
bfe3349b	232	static inline long long atomic64_add_return(long long i, atomic64_t *v)
1da177e4	233	{
0ccc8b7a HC	234	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i;
	235	}
	236
	237	static inline void atomic64_add(long long i, atomic64_t *v)
	238	{
	239	#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	240	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
	241	asm volatile(
	242	"agsi %0,%1\n"
	243	: "+Q" (v->counter)
	244	: "i" (i)
	245	: "cc", "memory");
	246	return;
	247	}
	248	#endif
	249	__ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER);
1da177e4	250	}
973bd993	251
bfe3349b	252	static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)
1da177e4	253	{
0ccc8b7a	254	__ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND, __ATOMIC64_NO_BARRIER);
1da177e4	255	}
973bd993	256
bfe3349b	257	static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)
1da177e4	258	{
0ccc8b7a	259	__ATOMIC64_LOOP(v, mask, __ATOMIC64_OR, __ATOMIC64_NO_BARRIER);
1da177e4 LT	260	}
1da177e4 LT	261
3a5f10e3 MD	262	#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
3a5f10e3 MD	263
bfe3349b	264	static inline long long atomic64_cmpxchg(atomic64_t *v,
973bd993 MS	265	long long old, long long new)
973bd993 MS	266	{
94c12cc7 MS	267	asm volatile(
94c12cc7 MS	268	" csg %0,%2,%1"
9a70a428 HC	269	: "+d" (old), "+Q" (v->counter)
9a70a428 HC	270	: "d" (new)
94c12cc7	271	: "cc", "memory");
973bd993 MS	272	return old;
973bd993 MS	273	}
1da177e4	274
75287430	275	#undef __ATOMIC64_LOOP
12751058 HC	276
	277	#else /* CONFIG_64BIT */
	278
	279	typedef struct {
	280	long long counter;
	281	} atomic64_t;
	282
	283	static inline long long atomic64_read(const atomic64_t *v)
	284	{
	285	register_pair rp;
	286
	287	asm volatile(
987bcdac MS	288	" lm %0,%N0,%1"
987bcdac MS	289	: "=&d" (rp) : "Q" (v->counter) );
12751058 HC	290	return rp.pair;
	291	}
	292
	293	static inline void atomic64_set(atomic64_t *v, long long i)
	294	{
	295	register_pair rp = {.pair = i};
	296
	297	asm volatile(
987bcdac MS	298	" stm %1,%N1,%0"
987bcdac MS	299	: "=Q" (v->counter) : "d" (rp) );
12751058 HC	300	}
	301
	302	static inline long long atomic64_xchg(atomic64_t *v, long long new)
	303	{
	304	register_pair rp_new = {.pair = new};
	305	register_pair rp_old;
	306
	307	asm volatile(
987bcdac MS	308	" lm %0,%N0,%1\n"
987bcdac MS	309	"0: cds %0,%2,%1\n"
12751058	310	" jl 0b\n"
9a70a428 HC	311	: "=&d" (rp_old), "+Q" (v->counter)
9a70a428 HC	312	: "d" (rp_new)
12751058 HC	313	: "cc");
	314	return rp_old.pair;
	315	}
	316
	317	static inline long long atomic64_cmpxchg(atomic64_t *v,
	318	long long old, long long new)
	319	{
	320	register_pair rp_old = {.pair = old};
	321	register_pair rp_new = {.pair = new};
	322
	323	asm volatile(
987bcdac	324	" cds %0,%2,%1"
9a70a428 HC	325	: "+&d" (rp_old), "+Q" (v->counter)
9a70a428 HC	326	: "d" (rp_new)
12751058 HC	327	: "cc");
	328	return rp_old.pair;
	329	}
	330
	331
	332	static inline long long atomic64_add_return(long long i, atomic64_t *v)
	333	{
	334	long long old, new;
	335
	336	do {
	337	old = atomic64_read(v);
	338	new = old + i;
	339	} while (atomic64_cmpxchg(v, old, new) != old);
	340	return new;
	341	}
	342
12751058 HC	343	static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v)
	344	{
	345	long long old, new;
	346
	347	do {
	348	old = atomic64_read(v);
	349	new = old \| mask;
	350	} while (atomic64_cmpxchg(v, old, new) != old);
	351	}
	352
	353	static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v)
	354	{
	355	long long old, new;
	356
	357	do {
	358	old = atomic64_read(v);
	359	new = old & mask;
	360	} while (atomic64_cmpxchg(v, old, new) != old);
	361	}
	362
5692e4d1 HC	363	static inline void atomic64_add(long long i, atomic64_t *v)
5692e4d1 HC	364	{
5692e4d1	365	atomic64_add_return(i, v);
5692e4d1 HC	366	}
5692e4d1 HC	367
0ccc8b7a HC	368	#endif /* CONFIG_64BIT */
0ccc8b7a HC	369
9a70a428	370	static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)
1da177e4	371	{
973bd993	372	long long c, old;
2ddb3ec4	373
973bd993	374	c = atomic64_read(v);
0b2fcfdb NP	375	for (;;) {
	376	if (unlikely(c == u))
	377	break;
9a70a428	378	old = atomic64_cmpxchg(v, c, c + i);
0b2fcfdb NP	379	if (likely(old == c))
0b2fcfdb NP	380	break;
973bd993	381	c = old;
0b2fcfdb	382	}
973bd993	383	return c != u;
1da177e4 LT	384	}
1da177e4 LT	385
2ddb3ec4 HC	386	static inline long long atomic64_dec_if_positive(atomic64_t *v)
	387	{
	388	long long c, old, dec;
	389
	390	c = atomic64_read(v);
	391	for (;;) {
	392	dec = c - 1;
	393	if (unlikely(dec < 0))
	394	break;
	395	old = atomic64_cmpxchg((v), c, dec);
	396	if (likely(old == c))
	397	break;
	398	c = old;
	399	}
	400	return dec;
	401	}
	402
12751058	403	#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0)
5692e4d1	404	#define atomic64_inc(_v) atomic64_add(1, _v)
12751058 HC	405	#define atomic64_inc_return(_v) atomic64_add_return(1, _v)
12751058 HC	406	#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0)
86d51bc3	407	#define atomic64_sub_return(_i, _v) atomic64_add_return(-(long long)(_i), _v)
5692e4d1	408	#define atomic64_sub(_i, _v) atomic64_add(-(long long)(_i), _v)
12751058	409	#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0)
5692e4d1	410	#define atomic64_dec(_v) atomic64_sub(1, _v)
12751058 HC	411	#define atomic64_dec_return(_v) atomic64_sub_return(1, _v)
	412	#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0)
	413	#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
8426e1f6	414
1da177e4 LT	415	#define smp_mb__before_atomic_dec() smp_mb()
	416	#define smp_mb__after_atomic_dec() smp_mb()
	417	#define smp_mb__before_atomic_inc() smp_mb()
	418	#define smp_mb__after_atomic_inc() smp_mb()
	419
1da177e4	420	#endif /* __ARCH_S390_ATOMIC__ */