[deliverable/linux.git] / arch / s390 / include / asm / bitops.h

/*
 *    Copyright IBM Corp. 1999,2013
 *
 *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
 *
 * The description below was taken in large parts from the powerpc
 * bitops header file:
 * Within a word, bits are numbered LSB first.  Lot's of places make
 * this assumption by directly testing bits with (val & (1<<nr)).
 * This can cause confusion for large (> 1 word) bitmaps on a
 * big-endian system because, unlike little endian, the number of each
 * bit depends on the word size.
 *
 * The bitop functions are defined to work on unsigned longs, so for an
 * s390x system the bits end up numbered:
 *   |63..............0|127............64|191...........128|255...........192|
 * and on s390:
 *   |31.....0|63....32|95....64|127...96|159..128|191..160|223..192|255..224|
 *
 * There are a few little-endian macros used mostly for filesystem
 * bitmaps, these work on similar bit arrays layouts, but
 * byte-oriented:
 *   |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56|
 *
 * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit
 * number field needs to be reversed compared to the big-endian bit
 * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b).
 *
 * We also have special functions which work with an MSB0 encoding:
 * on an s390x system the bits are numbered:
 *   |0..............63|64............127|128...........191|192...........255|
 * and on s390:
 *   |0.....31|32....63|64....95|96...127|128..159|160..191|192..223|224..255|
 *
 * The main difference is that bit 0-63 (64b) or 0-31 (32b) in the bit
 * number field needs to be reversed compared to the LSB0 encoded bit
 * fields. This can be achieved by XOR with 0x3f (64b) or 0x1f (32b).
 *
 */

#ifndef _S390_BITOPS_H
#define _S390_BITOPS_H

#ifndef _LINUX_BITOPS_H
#error only <linux/bitops.h> can be included directly
#endif

#include <linux/typecheck.h>
#include <linux/compiler.h>
#include <asm/barrier.h>

#define __BITOPS_NO_BARRIER	"\n"

#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES

#define __BITOPS_OR		"laog"
#define __BITOPS_AND		"lang"
#define __BITOPS_XOR		"laxg"
#define __BITOPS_BARRIER	"bcr	14,0\n"

#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
({								\
	unsigned long __old;					\
								\
	typecheck(unsigned long *, (__addr));			\
	asm volatile(						\
		__op_string "	%0,%2,%1\n"			\
		__barrier					\
		: "=d" (__old),	"+Q" (*(__addr))		\
		: "d" (__val)					\
		: "cc", "memory");				\
	__old;							\
})

#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */

#define __BITOPS_OR		"ogr"
#define __BITOPS_AND		"ngr"
#define __BITOPS_XOR		"xgr"
#define __BITOPS_BARRIER	"\n"

#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
({								\
	unsigned long __old, __new;				\
								\
	typecheck(unsigned long *, (__addr));			\
	asm volatile(						\
		"	lg	%0,%2\n"			\
		"0:	lgr	%1,%0\n"			\
		__op_string "	%1,%3\n"			\
		"	csg	%0,%1,%2\n"			\
		"	jl	0b"				\
		: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
		: "d" (__val)					\
		: "cc", "memory");				\
	__old;							\
})

#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */

#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)

static inline unsigned long *
__bitops_word(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long addr;

	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
	return (unsigned long *)addr;
}

static inline unsigned char *
__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
{
	return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
}

static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"oi	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (1 << (nr & 7))
			: "cc", "memory");
		return;
	}
#endif
	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	__BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER);
}

static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"ni	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (~(1 << (nr & 7)))
			: "cc", "memory");
		return;
	}
#endif
	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	__BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER);
}

static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"xi	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (1 << (nr & 7))
			: "cc", "memory");
		return;
	}
#endif
	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	__BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER);
}

static inline int
test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER);
	return (old & mask) != 0;
}

static inline int
test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER);
	return (old & ~mask) != 0;
}

static inline int
test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER);
	return (old & mask) != 0;
}

static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr |= 1 << (nr & 7);
}

static inline void 
__clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr &= ~(1 << (nr & 7));
}

static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr ^= 1 << (nr & 7);
}

static inline int
__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr |= 1 << (nr & 7);
	return (ch >> (nr & 7)) & 1;
}

static inline int
__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr &= ~(1 << (nr & 7));
	return (ch >> (nr & 7)) & 1;
}

static inline int
__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr ^= 1 << (nr & 7);
	return (ch >> (nr & 7)) & 1;
}

static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
{
	const volatile unsigned char *addr;

	addr = ((const volatile unsigned char *)ptr);
	addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
	return (*addr >> (nr & 7)) & 1;
}

static inline int test_and_set_bit_lock(unsigned long nr,
					volatile unsigned long *ptr)
{
	if (test_bit(nr, ptr))
		return 1;
	return test_and_set_bit(nr, ptr);
}

static inline void clear_bit_unlock(unsigned long nr,
				    volatile unsigned long *ptr)
{
	smp_mb__before_atomic();
	clear_bit(nr, ptr);
}

static inline void __clear_bit_unlock(unsigned long nr,
				      volatile unsigned long *ptr)
{
	smp_mb();
	__clear_bit(nr, ptr);
}

/*
 * Functions which use MSB0 bit numbering.
 * On an s390x system the bits are numbered:
 *   |0..............63|64............127|128...........191|192...........255|
 * and on s390:
 *   |0.....31|32....63|64....95|96...127|128..159|160..191|192..223|224..255|
 */
unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
				unsigned long offset);

static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline int test_bit_inv(unsigned long nr,
			       const volatile unsigned long *ptr)
{
	return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES

/**
 * __flogr - find leftmost one
 * @word - The word to search
 *
 * Returns the bit number of the most significant bit set,
 * where the most significant bit has bit number 0.
 * If no bit is set this function returns 64.
 */
static inline unsigned char __flogr(unsigned long word)
{
	if (__builtin_constant_p(word)) {
		unsigned long bit = 0;

		if (!word)
			return 64;
		if (!(word & 0xffffffff00000000UL)) {
			word <<= 32;
			bit += 32;
		}
		if (!(word & 0xffff000000000000UL)) {
			word <<= 16;
			bit += 16;
		}
		if (!(word & 0xff00000000000000UL)) {
			word <<= 8;
			bit += 8;
		}
		if (!(word & 0xf000000000000000UL)) {
			word <<= 4;
			bit += 4;
		}
		if (!(word & 0xc000000000000000UL)) {
			word <<= 2;
			bit += 2;
		}
		if (!(word & 0x8000000000000000UL)) {
			word <<= 1;
			bit += 1;
		}
		return bit;
	} else {
		register unsigned long bit asm("4") = word;
		register unsigned long out asm("5");

		asm volatile(
			"       flogr   %[bit],%[bit]\n"
			: [bit] "+d" (bit), [out] "=d" (out) : : "cc");
		return bit;
	}
}

/**
 * __ffs - find first bit in word.
 * @word: The word to search
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
static inline unsigned long __ffs(unsigned long word)
{
	return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
}

/**
 * ffs - find first bit set
 * @word: the word to search
 *
 * This is defined the same way as the libc and
 * compiler builtin ffs routines (man ffs).
 */
static inline int ffs(int word)
{
	unsigned long mask = 2 * BITS_PER_LONG - 1;
	unsigned int val = (unsigned int)word;

	return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
}

/**
 * __fls - find last (most-significant) set bit in a long word
 * @word: the word to search
 *
 * Undefined if no set bit exists, so code should check against 0 first.
 */
static inline unsigned long __fls(unsigned long word)
{
	return __flogr(word) ^ (BITS_PER_LONG - 1);
}

/**
 * fls64 - find last set bit in a 64-bit word
 * @word: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffsll, but returns the position of the most significant set bit.
 *
 * fls64(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 64.
 */
static inline int fls64(unsigned long word)
{
	unsigned long mask = 2 * BITS_PER_LONG - 1;

	return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
}

/**
 * fls - find last (most-significant) bit set
 * @word: the word to search
 *
 * This is defined the same way as ffs.
 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 */
static inline int fls(int word)
{
	return fls64((unsigned int)word);
}

#else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */

#include <asm-generic/bitops/__ffs.h>
#include <asm-generic/bitops/ffs.h>
#include <asm-generic/bitops/__fls.h>
#include <asm-generic/bitops/fls.h>
#include <asm-generic/bitops/fls64.h>

#endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */

#include <asm-generic/bitops/ffz.h>
#include <asm-generic/bitops/find.h>
#include <asm-generic/bitops/hweight.h>
#include <asm-generic/bitops/sched.h>
#include <asm-generic/bitops/le.h>
#include <asm-generic/bitops/ext2-atomic-setbit.h>

#endif /* _S390_BITOPS_H */
Commit	Line	Data
1da177e4	1	/*
746479cd	2	* Copyright IBM Corp. 1999,2013
1da177e4	3	*
746479cd HC	4	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
	5	*
	6	* The description below was taken in large parts from the powerpc
	7	* bitops header file:
	8	* Within a word, bits are numbered LSB first. Lot's of places make
	9	* this assumption by directly testing bits with (val & (1<<nr)).
	10	* This can cause confusion for large (> 1 word) bitmaps on a
	11	* big-endian system because, unlike little endian, the number of each
	12	* bit depends on the word size.
	13	*
	14	* The bitop functions are defined to work on unsigned longs, so for an
	15	* s390x system the bits end up numbered:
db85eaeb	16	* \|63..............0\|127............64\|191...........128\|255...........192\|
746479cd	17	* and on s390:
db85eaeb	18	* \|31.....0\|63....32\|95....64\|127...96\|159..128\|191..160\|223..192\|255..224\|
746479cd HC	19	*
	20	* There are a few little-endian macros used mostly for filesystem
	21	* bitmaps, these work on similar bit arrays layouts, but
	22	* byte-oriented:
	23	* \|7...0\|15...8\|23...16\|31...24\|39...32\|47...40\|55...48\|63...56\|
	24	*
	25	* The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit
	26	* number field needs to be reversed compared to the big-endian bit
	27	* fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b).
	28	*
	29	* We also have special functions which work with an MSB0 encoding:
	30	* on an s390x system the bits are numbered:
	31	* \|0..............63\|64............127\|128...........191\|192...........255\|
	32	* and on s390:
db85eaeb	33	* \|0.....31\|32....63\|64....95\|96...127\|128..159\|160..191\|192..223\|224..255\|
746479cd HC	34	*
	35	* The main difference is that bit 0-63 (64b) or 0-31 (32b) in the bit
	36	* number field needs to be reversed compared to the LSB0 encoded bit
	37	* fields. This can be achieved by XOR with 0x3f (64b) or 0x1f (32b).
1da177e4 LT	38	*
1da177e4 LT	39	*/
c406abd3	40
a53c8fab HC	41	#ifndef _S390_BITOPS_H
	42	#define _S390_BITOPS_H
	43
0624517d JS	44	#ifndef _LINUX_BITOPS_H
	45	#error only <linux/bitops.h> can be included directly
	46	#endif
	47
370b0b5f	48	#include <linux/typecheck.h>
1da177e4	49	#include <linux/compiler.h>
0ccc8b7a HC	50	#include <asm/barrier.h>
	51
	52	#define __BITOPS_NO_BARRIER "\n"
1da177e4	53
e344e52c HC	54	#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
	55
	56	#define __BITOPS_OR "laog"
	57	#define __BITOPS_AND "lang"
	58	#define __BITOPS_XOR "laxg"
0ccc8b7a	59	#define __BITOPS_BARRIER "bcr 14,0\n"
e344e52c	60
0ccc8b7a	61	#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier) \
e344e52c HC	62	({ \
	63	unsigned long __old; \
	64	\
370b0b5f	65	typecheck(unsigned long *, (__addr)); \
e344e52c HC	66	asm volatile( \
e344e52c HC	67	__op_string " %0,%2,%1\n" \
0ccc8b7a	68	__barrier \
370b0b5f	69	: "=d" (__old), "+Q" (*(__addr)) \
e344e52c	70	: "d" (__val) \
0ccc8b7a	71	: "cc", "memory"); \
e344e52c HC	72	__old; \
	73	})
	74
	75	#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
	76
1da177e4 LT	77	#define __BITOPS_OR "ogr"
	78	#define __BITOPS_AND "ngr"
	79	#define __BITOPS_XOR "xgr"
0ccc8b7a	80	#define __BITOPS_BARRIER "\n"
1da177e4	81
0ccc8b7a	82	#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier) \
e344e52c HC	83	({ \
	84	unsigned long __old, __new; \
	85	\
370b0b5f	86	typecheck(unsigned long *, (__addr)); \
94c12cc7 MS	87	asm volatile( \
	88	" lg %0,%2\n" \
	89	"0: lgr %1,%0\n" \
	90	__op_string " %1,%3\n" \
	91	" csg %0,%1,%2\n" \
	92	" jl 0b" \
370b0b5f HC	93	: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
370b0b5f HC	94	: "d" (__val) \
0ccc8b7a	95	: "cc", "memory"); \
e344e52c HC	96	__old; \
	97	})
	98
	99	#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
94c12cc7	100
01c2475f	101	#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
1da177e4	102
370b0b5f HC	103	static inline unsigned long *
	104	__bitops_word(unsigned long nr, volatile unsigned long *ptr)
	105	{
	106	unsigned long addr;
	107
	108	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
	109	return (unsigned long *)addr;
	110	}
	111
	112	static inline unsigned char *
	113	__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
	114	{
	115	return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
	116	}
	117
	118	static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	119	{
370b0b5f HC	120	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	121	unsigned long mask;
1da177e4	122
4ae80325 HC	123	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	124	if (__builtin_constant_p(nr)) {
	125	unsigned char *caddr = __bitops_byte(nr, ptr);
	126
	127	asm volatile(
	128	"oi %0,%b1\n"
	129	: "+Q" (*caddr)
	130	: "i" (1 << (nr & 7))
0ccc8b7a	131	: "cc", "memory");
4ae80325 HC	132	return;
	133	}
	134	#endif
01c2475f	135	mask = 1UL << (nr & (BITS_PER_LONG - 1));
0ccc8b7a	136	__BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER);
1da177e4 LT	137	}
1da177e4 LT	138
370b0b5f	139	static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	140	{
370b0b5f HC	141	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	142	unsigned long mask;
1da177e4	143
4ae80325 HC	144	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	145	if (__builtin_constant_p(nr)) {
	146	unsigned char *caddr = __bitops_byte(nr, ptr);
	147
	148	asm volatile(
	149	"ni %0,%b1\n"
	150	: "+Q" (*caddr)
	151	: "i" (~(1 << (nr & 7)))
0ccc8b7a	152	: "cc", "memory");
4ae80325 HC	153	return;
	154	}
	155	#endif
01c2475f	156	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
0ccc8b7a	157	__BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER);
1da177e4 LT	158	}
1da177e4 LT	159
370b0b5f	160	static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	161	{
370b0b5f HC	162	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	163	unsigned long mask;
1da177e4	164
4ae80325 HC	165	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	166	if (__builtin_constant_p(nr)) {
	167	unsigned char *caddr = __bitops_byte(nr, ptr);
	168
	169	asm volatile(
	170	"xi %0,%b1\n"
	171	: "+Q" (*caddr)
	172	: "i" (1 << (nr & 7))
0ccc8b7a	173	: "cc", "memory");
4ae80325 HC	174	return;
	175	}
	176	#endif
01c2475f	177	mask = 1UL << (nr & (BITS_PER_LONG - 1));
0ccc8b7a	178	__BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER);
1da177e4 LT	179	}
1da177e4 LT	180
1da177e4	181	static inline int
370b0b5f	182	test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	183	{
370b0b5f HC	184	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	185	unsigned long old, mask;
1da177e4	186
01c2475f	187	mask = 1UL << (nr & (BITS_PER_LONG - 1));
0ccc8b7a	188	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER);
1da177e4 LT	189	return (old & mask) != 0;
	190	}
	191
1da177e4	192	static inline int
370b0b5f	193	test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	194	{
370b0b5f HC	195	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	196	unsigned long old, mask;
1da177e4	197
01c2475f	198	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
0ccc8b7a	199	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER);
e344e52c	200	return (old & ~mask) != 0;
1da177e4 LT	201	}
1da177e4 LT	202
1da177e4	203	static inline int
370b0b5f	204	test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	205	{
370b0b5f HC	206	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	207	unsigned long old, mask;
1da177e4	208
01c2475f	209	mask = 1UL << (nr & (BITS_PER_LONG - 1));
0ccc8b7a	210	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER);
1da177e4 LT	211	return (old & mask) != 0;
1da177e4 LT	212	}
1da177e4	213
1da177e4 LT	214	static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4 LT	215	{
370b0b5f	216	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4	217
370b0b5f	218	*addr \|= 1 << (nr & 7);
1da177e4 LT	219	}
1da177e4 LT	220
1da177e4 LT	221	static inline void
	222	__clear_bit(unsigned long nr, volatile unsigned long *ptr)
	223	{
370b0b5f	224	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4	225
370b0b5f	226	*addr &= ~(1 << (nr & 7));
1da177e4 LT	227	}
1da177e4 LT	228
1da177e4 LT	229	static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4 LT	230	{
370b0b5f	231	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4	232
370b0b5f	233	*addr ^= 1 << (nr & 7);
1da177e4 LT	234	}
1da177e4 LT	235
1da177e4	236	static inline int
370b0b5f	237	__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	238	{
370b0b5f	239	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4 LT	240	unsigned char ch;
1da177e4 LT	241
370b0b5f HC	242	ch = *addr;
370b0b5f HC	243	*addr \|= 1 << (nr & 7);
1da177e4 LT	244	return (ch >> (nr & 7)) & 1;
1da177e4 LT	245	}
1da177e4	246
1da177e4	247	static inline int
370b0b5f	248	__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	249	{
370b0b5f	250	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4 LT	251	unsigned char ch;
1da177e4 LT	252
370b0b5f HC	253	ch = *addr;
370b0b5f HC	254	*addr &= ~(1 << (nr & 7));
1da177e4 LT	255	return (ch >> (nr & 7)) & 1;
1da177e4 LT	256	}
1da177e4	257
1da177e4	258	static inline int
370b0b5f	259	__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	260	{
370b0b5f	261	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4 LT	262	unsigned char ch;
1da177e4 LT	263
370b0b5f HC	264	ch = *addr;
370b0b5f HC	265	*addr ^= 1 << (nr & 7);
1da177e4 LT	266	return (ch >> (nr & 7)) & 1;
1da177e4 LT	267	}
1da177e4	268
370b0b5f	269	static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
1da177e4	270	{
370b0b5f	271	const volatile unsigned char *addr;
1da177e4	272
370b0b5f HC	273	addr = ((const volatile unsigned char *)ptr);
	274	addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
	275	return (*addr >> (nr & 7)) & 1;
1da177e4 LT	276	}
1da177e4 LT	277
acdc9fc9 MS	278	static inline int test_and_set_bit_lock(unsigned long nr,
	279	volatile unsigned long *ptr)
	280	{
	281	if (test_bit(nr, ptr))
	282	return 1;
	283	return test_and_set_bit(nr, ptr);
	284	}
	285
	286	static inline void clear_bit_unlock(unsigned long nr,
	287	volatile unsigned long *ptr)
	288	{
	289	smp_mb__before_atomic();
	290	clear_bit(nr, ptr);
	291	}
	292
	293	static inline void __clear_bit_unlock(unsigned long nr,
	294	volatile unsigned long *ptr)
	295	{
	296	smp_mb();
	297	__clear_bit(nr, ptr);
	298	}
	299
afff7e2b	300	/*
7d7c7b24 HC	301	* Functions which use MSB0 bit numbering.
	302	* On an s390x system the bits are numbered:
	303	* \|0..............63\|64............127\|128...........191\|192...........255\|
	304	* and on s390:
db85eaeb	305	* \|0.....31\|32....63\|64....95\|96...127\|128..159\|160..191\|192..223\|224..255\|
e56e4e87	306	*/
7d7c7b24 HC	307	unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
	308	unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
	309	unsigned long offset);
	310
	311	static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	312	{
	313	return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	314	}
	315
	316	static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	317	{
	318	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	319	}
	320
	321	static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	322	{
	323	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	324	}
	325
	326	static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	327	{
	328	return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	329	}
	330
	331	static inline int test_bit_inv(unsigned long nr,
	332	const volatile unsigned long *ptr)
	333	{
	334	return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	335	}
e56e4e87	336
b1cb7e2b HC	337	#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
	338
	339	/**
	340	* __flogr - find leftmost one
	341	* @word - The word to search
	342	*
	343	* Returns the bit number of the most significant bit set,
	344	* where the most significant bit has bit number 0.
	345	* If no bit is set this function returns 64.
	346	*/
	347	static inline unsigned char __flogr(unsigned long word)
	348	{
	349	if (__builtin_constant_p(word)) {
	350	unsigned long bit = 0;
	351
	352	if (!word)
	353	return 64;
	354	if (!(word & 0xffffffff00000000UL)) {
	355	word <<= 32;
	356	bit += 32;
	357	}
	358	if (!(word & 0xffff000000000000UL)) {
	359	word <<= 16;
	360	bit += 16;
	361	}
	362	if (!(word & 0xff00000000000000UL)) {
	363	word <<= 8;
	364	bit += 8;
	365	}
	366	if (!(word & 0xf000000000000000UL)) {
	367	word <<= 4;
	368	bit += 4;
	369	}
	370	if (!(word & 0xc000000000000000UL)) {
	371	word <<= 2;
	372	bit += 2;
	373	}
	374	if (!(word & 0x8000000000000000UL)) {
	375	word <<= 1;
	376	bit += 1;
	377	}
	378	return bit;
	379	} else {
	380	register unsigned long bit asm("4") = word;
	381	register unsigned long out asm("5");
	382
	383	asm volatile(
	384	" flogr %[bit],%[bit]\n"
	385	: [bit] "+d" (bit), [out] "=d" (out) : : "cc");
	386	return bit;
	387	}
	388	}
	389
	390	/**
	391	* __ffs - find first bit in word.
	392	* @word: The word to search
	393	*
	394	* Undefined if no bit exists, so code should check against 0 first.
	395	*/
	396	static inline unsigned long __ffs(unsigned long word)
	397	{
	398	return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
	399	}
	400
401	/**
402	* ffs - find first bit set
403	* @word: the word to search
404	*
405	* This is defined the same way as the libc and
406	* compiler builtin ffs routines (man ffs).
407	*/
408	static inline int ffs(int word)
409	{
410	unsigned long mask = 2 * BITS_PER_LONG - 1;
411	unsigned int val = (unsigned int)word;
412
413	return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
414	}
415
416	/**
417	* __fls - find last (most-significant) set bit in a long word
418	* @word: the word to search
419	*
420	* Undefined if no set bit exists, so code should check against 0 first.
421	*/
422	static inline unsigned long __fls(unsigned long word)
423	{
424	return __flogr(word) ^ (BITS_PER_LONG - 1);
425	}
426
427	/**
428	* fls64 - find last set bit in a 64-bit word
429	* @word: the word to search
430	*
431	* This is defined in a similar way as the libc and compiler builtin
432	* ffsll, but returns the position of the most significant set bit.
433	*
434	* fls64(value) returns 0 if value is 0 or the position of the last
435	* set bit if value is nonzero. The last (most significant) bit is
436	* at position 64.
437	*/
438	static inline int fls64(unsigned long word)
439	{
440	unsigned long mask = 2 * BITS_PER_LONG - 1;
441
442	return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
443	}
444
445	/**
446	* fls - find last (most-significant) bit set
447	* @word: the word to search
448	*
449	* This is defined the same way as ffs.
450	* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
451	*/
452	static inline int fls(int word)
453	{
454	return fls64((unsigned int)word);
455	}
456
457	#else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
458
746479cd HC	459	#include <asm-generic/bitops/__ffs.h>
746479cd HC	460	#include <asm-generic/bitops/ffs.h>
56a6b1eb	461	#include <asm-generic/bitops/__fls.h>
746479cd	462	#include <asm-generic/bitops/fls.h>
7e33db4e	463	#include <asm-generic/bitops/fls64.h>
b1cb7e2b HC	464
	465	#endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
	466
746479cd HC	467	#include <asm-generic/bitops/ffz.h>
746479cd HC	468	#include <asm-generic/bitops/find.h>
7e33db4e	469	#include <asm-generic/bitops/hweight.h>
746479cd	470	#include <asm-generic/bitops/sched.h>
802caabb	471	#include <asm-generic/bitops/le.h>
148817ba	472	#include <asm-generic/bitops/ext2-atomic-setbit.h>
67fe9251	473
1da177e4	474	#endif /* _S390_BITOPS_H */