[deliverable/linux.git] / arch / x86 / include / asm / bitops.h

#ifndef _ASM_X86_BITOPS_H
#define _ASM_X86_BITOPS_H

/*
 * Copyright 1992, Linus Torvalds.
 */

#ifndef _LINUX_BITOPS_H
#error only <linux/bitops.h> can be included directly
#endif

#include <linux/compiler.h>
#include <asm/alternative.h>

/*
 * These have to be done with inline assembly: that way the bit-setting
 * is guaranteed to be atomic. All bit operations return 0 if the bit
 * was cleared before the operation and != 0 if it was not.
 *
 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
 */

#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
/* Technically wrong, but this avoids compilation errors on some gcc
   versions. */
#define BITOP_ADDR(x) "=m" (*(volatile long *) (x))
#else
#define BITOP_ADDR(x) "+m" (*(volatile long *) (x))
#endif

#define ADDR				BITOP_ADDR(addr)

/*
 * We do the locked ops that don't return the old value as
 * a mask operation on a byte.
 */
#define IS_IMMEDIATE(nr)		(__builtin_constant_p(nr))
#define CONST_MASK_ADDR(nr, addr)	BITOP_ADDR((void *)(addr) + ((nr)>>3))
#define CONST_MASK(nr)			(1 << ((nr) & 7))

/**
 * set_bit - Atomically set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * This function is atomic and may not be reordered.  See __set_bit()
 * if you do not require the atomic guarantees.
 *
 * Note: there are no guarantees that this function will not be reordered
 * on non x86 architectures, so if you are writing portable code,
 * make sure not to rely on its reordering guarantees.
 *
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static inline void set_bit(unsigned int nr, volatile unsigned long *addr)
{
	if (IS_IMMEDIATE(nr)) {
		asm volatile(LOCK_PREFIX "orb %1,%0"
			: CONST_MASK_ADDR(nr, addr)
			: "iq" ((u8)CONST_MASK(nr))
			: "memory");
	} else {
		asm volatile(LOCK_PREFIX "bts %1,%0"
			: BITOP_ADDR(addr) : "Ir" (nr) : "memory");
	}
}

/**
 * __set_bit - Set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * Unlike set_bit(), this function is non-atomic and may be reordered.
 * If it's called on the same region of memory simultaneously, the effect
 * may be that only one operation succeeds.
 */
static inline void __set_bit(int nr, volatile unsigned long *addr)
{
	asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory");
}

/**
 * clear_bit - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * clear_bit() is atomic and may not be reordered.  However, it does
 * not contain a memory barrier, so if it is used for locking purposes,
 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
 * in order to ensure changes are visible on other processors.
 */
static inline void clear_bit(int nr, volatile unsigned long *addr)
{
	if (IS_IMMEDIATE(nr)) {
		asm volatile(LOCK_PREFIX "andb %1,%0"
			: CONST_MASK_ADDR(nr, addr)
			: "iq" ((u8)~CONST_MASK(nr)));
	} else {
		asm volatile(LOCK_PREFIX "btr %1,%0"
			: BITOP_ADDR(addr)
			: "Ir" (nr));
	}
}

/*
 * clear_bit_unlock - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * clear_bit() is atomic and implies release semantics before the memory
 * operation. It can be used for an unlock.
 */
static inline void clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
{
	barrier();
	clear_bit(nr, addr);
}

static inline void __clear_bit(int nr, volatile unsigned long *addr)
{
	asm volatile("btr %1,%0" : ADDR : "Ir" (nr));
}

/*
 * __clear_bit_unlock - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * __clear_bit() is non-atomic and implies release semantics before the memory
 * operation. It can be used for an unlock if no other CPUs can concurrently
 * modify other bits in the word.
 *
 * No memory barrier is required here, because x86 cannot reorder stores past
 * older loads. Same principle as spin_unlock.
 */
static inline void __clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
{
	barrier();
	__clear_bit(nr, addr);
}

#define smp_mb__before_clear_bit()	barrier()
#define smp_mb__after_clear_bit()	barrier()

/**
 * __change_bit - Toggle a bit in memory
 * @nr: the bit to change
 * @addr: the address to start counting from
 *
 * Unlike change_bit(), this function is non-atomic and may be reordered.
 * If it's called on the same region of memory simultaneously, the effect
 * may be that only one operation succeeds.
 */
static inline void __change_bit(int nr, volatile unsigned long *addr)
{
	asm volatile("btc %1,%0" : ADDR : "Ir" (nr));
}

/**
 * change_bit - Toggle a bit in memory
 * @nr: Bit to change
 * @addr: Address to start counting from
 *
 * change_bit() is atomic and may not be reordered.
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static inline void change_bit(int nr, volatile unsigned long *addr)
{
	if (IS_IMMEDIATE(nr)) {
		asm volatile(LOCK_PREFIX "xorb %1,%0"
			: CONST_MASK_ADDR(nr, addr)
			: "iq" ((u8)CONST_MASK(nr)));
	} else {
		asm volatile(LOCK_PREFIX "btc %1,%0"
			: BITOP_ADDR(addr)
			: "Ir" (nr));
	}
}

/**
 * test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
{
	int oldbit;

	asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
		     "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");

	return oldbit;
}

/**
 * test_and_set_bit_lock - Set a bit and return its old value for lock
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This is the same as test_and_set_bit on x86.
 */
static inline int test_and_set_bit_lock(int nr, volatile unsigned long *addr)
{
	return test_and_set_bit(nr, addr);
}

/**
 * __test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is non-atomic and can be reordered.
 * If two examples of this operation race, one can appear to succeed
 * but actually fail.  You must protect multiple accesses with a lock.
 */
static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
{
	int oldbit;

	asm("bts %2,%1\n\t"
	    "sbb %0,%0"
	    : "=r" (oldbit), ADDR
	    : "Ir" (nr));
	return oldbit;
}

/**
 * test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)
{
	int oldbit;

	asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
		     "sbb %0,%0"
		     : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");

	return oldbit;
}

/**
 * __test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is non-atomic and can be reordered.
 * If two examples of this operation race, one can appear to succeed
 * but actually fail.  You must protect multiple accesses with a lock.
 */
static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
{
	int oldbit;

	asm volatile("btr %2,%1\n\t"
		     "sbb %0,%0"
		     : "=r" (oldbit), ADDR
		     : "Ir" (nr));
	return oldbit;
}

/* WARNING: non atomic and it can be reordered! */
static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
{
	int oldbit;

	asm volatile("btc %2,%1\n\t"
		     "sbb %0,%0"
		     : "=r" (oldbit), ADDR
		     : "Ir" (nr) : "memory");

	return oldbit;
}

/**
 * test_and_change_bit - Change a bit and return its old value
 * @nr: Bit to change
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
{
	int oldbit;

	asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
		     "sbb %0,%0"
		     : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");

	return oldbit;
}

static inline int constant_test_bit(int nr, const volatile unsigned long *addr)
{
	return ((1UL << (nr % BITS_PER_LONG)) &
		(((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
}

static inline int variable_test_bit(int nr, volatile const unsigned long *addr)
{
	int oldbit;

	asm volatile("bt %2,%1\n\t"
		     "sbb %0,%0"
		     : "=r" (oldbit)
		     : "m" (*(unsigned long *)addr), "Ir" (nr));

	return oldbit;
}

#if 0 /* Fool kernel-doc since it doesn't do macros yet */
/**
 * test_bit - Determine whether a bit is set
 * @nr: bit number to test
 * @addr: Address to start counting from
 */
static int test_bit(int nr, const volatile unsigned long *addr);
#endif

#define test_bit(nr, addr)			\
	(__builtin_constant_p((nr))		\
	 ? constant_test_bit((nr), (addr))	\
	 : variable_test_bit((nr), (addr)))

/**
 * __ffs - find first set bit in word
 * @word: The word to search
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
static inline unsigned long __ffs(unsigned long word)
{
	asm("bsf %1,%0"
		: "=r" (word)
		: "rm" (word));
	return word;
}

/**
 * ffz - find first zero bit in word
 * @word: The word to search
 *
 * Undefined if no zero exists, so code should check against ~0UL first.
 */
static inline unsigned long ffz(unsigned long word)
{
	asm("bsf %1,%0"
		: "=r" (word)
		: "r" (~word));
	return word;
}

/*
 * __fls: find last set bit in word
 * @word: The word to search
 *
 * Undefined if no set bit exists, so code should check against 0 first.
 */
static inline unsigned long __fls(unsigned long word)
{
	asm("bsr %1,%0"
	    : "=r" (word)
	    : "rm" (word));
	return word;
}

#ifdef __KERNEL__
/**
 * ffs - find first set bit in word
 * @x: the word to search
 *
 * This is defined the same way as the libc and compiler builtin ffs
 * routines, therefore differs in spirit from the other bitops.
 *
 * ffs(value) returns 0 if value is 0 or the position of the first
 * set bit if value is nonzero. The first (least significant) bit
 * is at position 1.
 */
static inline int ffs(int x)
{
	int r;
#ifdef CONFIG_X86_CMOV
	asm("bsfl %1,%0\n\t"
	    "cmovzl %2,%0"
	    : "=r" (r) : "rm" (x), "r" (-1));
#else
	asm("bsfl %1,%0\n\t"
	    "jnz 1f\n\t"
	    "movl $-1,%0\n"
	    "1:" : "=r" (r) : "rm" (x));
#endif
	return r + 1;
}

/**
 * fls - find last set bit in word
 * @x: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffs, but returns the position of the most significant set bit.
 *
 * fls(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 32.
 */
static inline int fls(int x)
{
	int r;
#ifdef CONFIG_X86_CMOV
	asm("bsrl %1,%0\n\t"
	    "cmovzl %2,%0"
	    : "=&r" (r) : "rm" (x), "rm" (-1));
#else
	asm("bsrl %1,%0\n\t"
	    "jnz 1f\n\t"
	    "movl $-1,%0\n"
	    "1:" : "=r" (r) : "rm" (x));
#endif
	return r + 1;
}
#endif /* __KERNEL__ */

#undef ADDR

#ifdef __KERNEL__

#include <asm-generic/bitops/sched.h>

#define ARCH_HAS_FAST_MULTIPLIER 1

#include <asm-generic/bitops/hweight.h>

#endif /* __KERNEL__ */

#include <asm-generic/bitops/fls64.h>

#ifdef __KERNEL__

#include <asm-generic/bitops/ext2-non-atomic.h>

#define ext2_set_bit_atomic(lock, nr, addr)			\
	test_and_set_bit((nr), (unsigned long *)(addr))
#define ext2_clear_bit_atomic(lock, nr, addr)			\
	test_and_clear_bit((nr), (unsigned long *)(addr))

#include <asm-generic/bitops/minix.h>

#endif /* __KERNEL__ */
#endif /* _ASM_X86_BITOPS_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_BITOPS_H
1965aae3 PA	2	#define _ASM_X86_BITOPS_H
1c54d770 JF	3
	4	/*
	5	* Copyright 1992, Linus Torvalds.
	6	*/
	7
	8	#ifndef _LINUX_BITOPS_H
	9	#error only <linux/bitops.h> can be included directly
	10	#endif
	11
	12	#include <linux/compiler.h>
	13	#include <asm/alternative.h>
	14
	15	/*
	16	* These have to be done with inline assembly: that way the bit-setting
	17	* is guaranteed to be atomic. All bit operations return 0 if the bit
	18	* was cleared before the operation and != 0 if it was not.
	19	*
	20	* bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
	21	*/
	22
	23	#if __GNUC__ < 4 \|\| (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
	24	/* Technically wrong, but this avoids compilation errors on some gcc
	25	versions. */
1a750e0c	26	#define BITOP_ADDR(x) "=m" ((volatile long ) (x))
1c54d770	27	#else
1a750e0c	28	#define BITOP_ADDR(x) "+m" ((volatile long ) (x))
1c54d770 JF	29	#endif
1c54d770 JF	30
7dbceaf9	31	#define ADDR BITOP_ADDR(addr)
1a750e0c LT	32
	33	/*
	34	* We do the locked ops that don't return the old value as
	35	* a mask operation on a byte.
	36	*/
7dbceaf9 IM	37	#define IS_IMMEDIATE(nr) (__builtin_constant_p(nr))
	38	#define CONST_MASK_ADDR(nr, addr) BITOP_ADDR((void *)(addr) + ((nr)>>3))
	39	#define CONST_MASK(nr) (1 << ((nr) & 7))
1a750e0c	40
1c54d770 JF	41	/**
	42	* set_bit - Atomically set a bit in memory
	43	* @nr: the bit to set
	44	* @addr: the address to start counting from
	45	*
	46	* This function is atomic and may not be reordered. See __set_bit()
	47	* if you do not require the atomic guarantees.
	48	*
	49	* Note: there are no guarantees that this function will not be reordered
	50	* on non x86 architectures, so if you are writing portable code,
	51	* make sure not to rely on its reordering guarantees.
	52	*
	53	* Note that @nr may be almost arbitrarily large; this function is not
	54	* restricted to acting on a single-word quantity.
	55	*/
1a750e0c	56	static inline void set_bit(unsigned int nr, volatile unsigned long *addr)
1c54d770	57	{
7dbceaf9 IM	58	if (IS_IMMEDIATE(nr)) {
	59	asm volatile(LOCK_PREFIX "orb %1,%0"
	60	: CONST_MASK_ADDR(nr, addr)
437a0a54	61	: "iq" ((u8)CONST_MASK(nr))
7dbceaf9 IM	62	: "memory");
	63	} else {
	64	asm volatile(LOCK_PREFIX "bts %1,%0"
	65	: BITOP_ADDR(addr) : "Ir" (nr) : "memory");
	66	}
1c54d770 JF	67	}
	68
	69	/**
	70	* __set_bit - Set a bit in memory
	71	* @nr: the bit to set
	72	* @addr: the address to start counting from
	73	*
	74	* Unlike set_bit(), this function is non-atomic and may be reordered.
	75	* If it's called on the same region of memory simultaneously, the effect
	76	* may be that only one operation succeeds.
	77	*/
5136dea5	78	static inline void __set_bit(int nr, volatile unsigned long *addr)
1c54d770	79	{
f19dcf4a	80	asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory");
1c54d770 JF	81	}
1c54d770 JF	82
1c54d770 JF	83	/**
	84	* clear_bit - Clears a bit in memory
	85	* @nr: Bit to clear
	86	* @addr: Address to start counting from
	87	*
	88	* clear_bit() is atomic and may not be reordered. However, it does
	89	* not contain a memory barrier, so if it is used for locking purposes,
	90	* you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
	91	* in order to ensure changes are visible on other processors.
	92	*/
5136dea5	93	static inline void clear_bit(int nr, volatile unsigned long *addr)
1c54d770	94	{
7dbceaf9 IM	95	if (IS_IMMEDIATE(nr)) {
	96	asm volatile(LOCK_PREFIX "andb %1,%0"
	97	: CONST_MASK_ADDR(nr, addr)
437a0a54	98	: "iq" ((u8)~CONST_MASK(nr)));
7dbceaf9 IM	99	} else {
	100	asm volatile(LOCK_PREFIX "btr %1,%0"
	101	: BITOP_ADDR(addr)
	102	: "Ir" (nr));
	103	}
1c54d770 JF	104	}
	105
	106	/*
	107	* clear_bit_unlock - Clears a bit in memory
	108	* @nr: Bit to clear
	109	* @addr: Address to start counting from
	110	*
	111	* clear_bit() is atomic and implies release semantics before the memory
	112	* operation. It can be used for an unlock.
	113	*/
5136dea5	114	static inline void clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
1c54d770 JF	115	{
	116	barrier();
	117	clear_bit(nr, addr);
	118	}
	119
5136dea5	120	static inline void __clear_bit(int nr, volatile unsigned long *addr)
1c54d770	121	{
eb2b4e68	122	asm volatile("btr %1,%0" : ADDR : "Ir" (nr));
1c54d770 JF	123	}
	124
	125	/*
	126	* __clear_bit_unlock - Clears a bit in memory
	127	* @nr: Bit to clear
	128	* @addr: Address to start counting from
	129	*
	130	* __clear_bit() is non-atomic and implies release semantics before the memory
	131	* operation. It can be used for an unlock if no other CPUs can concurrently
	132	* modify other bits in the word.
	133	*
	134	* No memory barrier is required here, because x86 cannot reorder stores past
	135	* older loads. Same principle as spin_unlock.
	136	*/
5136dea5	137	static inline void __clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
1c54d770 JF	138	{
	139	barrier();
	140	__clear_bit(nr, addr);
	141	}
	142
	143	#define smp_mb__before_clear_bit() barrier()
	144	#define smp_mb__after_clear_bit() barrier()
	145
	146	/**
	147	* __change_bit - Toggle a bit in memory
	148	* @nr: the bit to change
	149	* @addr: the address to start counting from
	150	*
	151	* Unlike change_bit(), this function is non-atomic and may be reordered.
	152	* If it's called on the same region of memory simultaneously, the effect
	153	* may be that only one operation succeeds.
	154	*/
5136dea5	155	static inline void __change_bit(int nr, volatile unsigned long *addr)
1c54d770	156	{
eb2b4e68	157	asm volatile("btc %1,%0" : ADDR : "Ir" (nr));
1c54d770 JF	158	}
	159
	160	/**
	161	* change_bit - Toggle a bit in memory
	162	* @nr: Bit to change
	163	* @addr: Address to start counting from
	164	*
	165	* change_bit() is atomic and may not be reordered.
	166	* Note that @nr may be almost arbitrarily large; this function is not
	167	* restricted to acting on a single-word quantity.
	168	*/
5136dea5	169	static inline void change_bit(int nr, volatile unsigned long *addr)
1c54d770	170	{
838e8bb7 UB	171	if (IS_IMMEDIATE(nr)) {
	172	asm volatile(LOCK_PREFIX "xorb %1,%0"
	173	: CONST_MASK_ADDR(nr, addr)
	174	: "iq" ((u8)CONST_MASK(nr)));
	175	} else {
	176	asm volatile(LOCK_PREFIX "btc %1,%0"
	177	: BITOP_ADDR(addr)
	178	: "Ir" (nr));
	179	}
1c54d770 JF	180	}
	181
	182	/**
	183	* test_and_set_bit - Set a bit and return its old value
	184	* @nr: Bit to set
	185	* @addr: Address to count from
	186	*
	187	* This operation is atomic and cannot be reordered.
	188	* It also implies a memory barrier.
	189	*/
5136dea5	190	static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
1c54d770 JF	191	{
	192	int oldbit;
	193
	194	asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
286275c9	195	"sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
1c54d770 JF	196
	197	return oldbit;
	198	}
	199
	200	/**
	201	* test_and_set_bit_lock - Set a bit and return its old value for lock
	202	* @nr: Bit to set
	203	* @addr: Address to count from
	204	*
	205	* This is the same as test_and_set_bit on x86.
	206	*/
5136dea5	207	static inline int test_and_set_bit_lock(int nr, volatile unsigned long *addr)
1c54d770 JF	208	{
	209	return test_and_set_bit(nr, addr);
	210	}
	211
	212	/**
	213	* __test_and_set_bit - Set a bit and return its old value
	214	* @nr: Bit to set
	215	* @addr: Address to count from
	216	*
	217	* This operation is non-atomic and can be reordered.
	218	* If two examples of this operation race, one can appear to succeed
	219	* but actually fail. You must protect multiple accesses with a lock.
	220	*/
5136dea5	221	static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
1c54d770 JF	222	{
	223	int oldbit;
	224
eb2b4e68 SHT	225	asm("bts %2,%1\n\t"
	226	"sbb %0,%0"
	227	: "=r" (oldbit), ADDR
	228	: "Ir" (nr));
1c54d770 JF	229	return oldbit;
	230	}
	231
	232	/**
	233	* test_and_clear_bit - Clear a bit and return its old value
	234	* @nr: Bit to clear
	235	* @addr: Address to count from
	236	*
	237	* This operation is atomic and cannot be reordered.
	238	* It also implies a memory barrier.
	239	*/
5136dea5	240	static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)
1c54d770 JF	241	{
	242	int oldbit;
	243
	244	asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
	245	"sbb %0,%0"
286275c9	246	: "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
1c54d770 JF	247
	248	return oldbit;
	249	}
	250
	251	/**
	252	* __test_and_clear_bit - Clear a bit and return its old value
	253	* @nr: Bit to clear
	254	* @addr: Address to count from
	255	*
	256	* This operation is non-atomic and can be reordered.
	257	* If two examples of this operation race, one can appear to succeed
	258	* but actually fail. You must protect multiple accesses with a lock.
	259	*/
5136dea5	260	static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
1c54d770 JF	261	{
	262	int oldbit;
	263
eb2b4e68	264	asm volatile("btr %2,%1\n\t"
1c54d770	265	"sbb %0,%0"
eb2b4e68 SHT	266	: "=r" (oldbit), ADDR
eb2b4e68 SHT	267	: "Ir" (nr));
1c54d770 JF	268	return oldbit;
	269	}
	270
	271	/* WARNING: non atomic and it can be reordered! */
5136dea5	272	static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
1c54d770 JF	273	{
	274	int oldbit;
	275
eb2b4e68	276	asm volatile("btc %2,%1\n\t"
1c54d770	277	"sbb %0,%0"
eb2b4e68 SHT	278	: "=r" (oldbit), ADDR
eb2b4e68 SHT	279	: "Ir" (nr) : "memory");
1c54d770 JF	280
	281	return oldbit;
	282	}
	283
	284	/**
	285	* test_and_change_bit - Change a bit and return its old value
	286	* @nr: Bit to change
	287	* @addr: Address to count from
	288	*
	289	* This operation is atomic and cannot be reordered.
	290	* It also implies a memory barrier.
	291	*/
5136dea5	292	static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
1c54d770 JF	293	{
	294	int oldbit;
	295
	296	asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
	297	"sbb %0,%0"
286275c9	298	: "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
1c54d770 JF	299
	300	return oldbit;
	301	}
	302
5136dea5	303	static inline int constant_test_bit(int nr, const volatile unsigned long *addr)
1c54d770	304	{
26996dd2 GOC	305	return ((1UL << (nr % BITS_PER_LONG)) &
26996dd2 GOC	306	(((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
1c54d770 JF	307	}
1c54d770 JF	308
5136dea5	309	static inline int variable_test_bit(int nr, volatile const unsigned long *addr)
1c54d770 JF	310	{
	311	int oldbit;
	312
eb2b4e68	313	asm volatile("bt %2,%1\n\t"
1c54d770 JF	314	"sbb %0,%0"
1c54d770 JF	315	: "=r" (oldbit)
eb2b4e68	316	: "m" ((unsigned long )addr), "Ir" (nr));
1c54d770 JF	317
	318	return oldbit;
	319	}
	320
	321	#if 0 /* Fool kernel-doc since it doesn't do macros yet */
	322	/**
	323	* test_bit - Determine whether a bit is set
	324	* @nr: bit number to test
	325	* @addr: Address to start counting from
	326	*/
	327	static int test_bit(int nr, const volatile unsigned long *addr);
	328	#endif
	329
f19dcf4a JP	330	#define test_bit(nr, addr) \
	331	(__builtin_constant_p((nr)) \
	332	? constant_test_bit((nr), (addr)) \
	333	: variable_test_bit((nr), (addr)))
1c54d770	334
12d9c842 AH	335	/**
	336	* __ffs - find first set bit in word
	337	* @word: The word to search
	338	*
	339	* Undefined if no bit exists, so code should check against 0 first.
	340	*/
	341	static inline unsigned long __ffs(unsigned long word)
	342	{
f19dcf4a JP	343	asm("bsf %1,%0"
	344	: "=r" (word)
	345	: "rm" (word));
12d9c842 AH	346	return word;
	347	}
	348
	349	/**
	350	* ffz - find first zero bit in word
	351	* @word: The word to search
	352	*
	353	* Undefined if no zero exists, so code should check against ~0UL first.
	354	*/
	355	static inline unsigned long ffz(unsigned long word)
	356	{
f19dcf4a JP	357	asm("bsf %1,%0"
	358	: "=r" (word)
	359	: "r" (~word));
12d9c842 AH	360	return word;
	361	}
	362
	363	/*
	364	* __fls: find last set bit in word
	365	* @word: The word to search
	366	*
8450e853	367	* Undefined if no set bit exists, so code should check against 0 first.
12d9c842 AH	368	*/
	369	static inline unsigned long __fls(unsigned long word)
	370	{
f19dcf4a JP	371	asm("bsr %1,%0"
	372	: "=r" (word)
	373	: "rm" (word));
12d9c842 AH	374	return word;
	375	}
	376
	377	#ifdef __KERNEL__
	378	/**
	379	* ffs - find first set bit in word
	380	* @x: the word to search
	381	*
	382	* This is defined the same way as the libc and compiler builtin ffs
	383	* routines, therefore differs in spirit from the other bitops.
	384	*
	385	* ffs(value) returns 0 if value is 0 or the position of the first
	386	* set bit if value is nonzero. The first (least significant) bit
	387	* is at position 1.
	388	*/
	389	static inline int ffs(int x)
	390	{
	391	int r;
	392	#ifdef CONFIG_X86_CMOV
f19dcf4a JP	393	asm("bsfl %1,%0\n\t"
	394	"cmovzl %2,%0"
	395	: "=r" (r) : "rm" (x), "r" (-1));
12d9c842	396	#else
f19dcf4a JP	397	asm("bsfl %1,%0\n\t"
	398	"jnz 1f\n\t"
	399	"movl $-1,%0\n"
	400	"1:" : "=r" (r) : "rm" (x));
12d9c842 AH	401	#endif
	402	return r + 1;
	403	}
	404
	405	/**
	406	* fls - find last set bit in word
	407	* @x: the word to search
	408	*
	409	* This is defined in a similar way as the libc and compiler builtin
	410	* ffs, but returns the position of the most significant set bit.
	411	*
	412	* fls(value) returns 0 if value is 0 or the position of the last
	413	* set bit if value is nonzero. The last (most significant) bit is
	414	* at position 32.
	415	*/
	416	static inline int fls(int x)
	417	{
	418	int r;
	419	#ifdef CONFIG_X86_CMOV
f19dcf4a JP	420	asm("bsrl %1,%0\n\t"
	421	"cmovzl %2,%0"
	422	: "=&r" (r) : "rm" (x), "rm" (-1));
12d9c842	423	#else
f19dcf4a JP	424	asm("bsrl %1,%0\n\t"
	425	"jnz 1f\n\t"
	426	"movl $-1,%0\n"
	427	"1:" : "=r" (r) : "rm" (x));
12d9c842 AH	428	#endif
	429	return r + 1;
	430	}
	431	#endif /* __KERNEL__ */
	432
1c54d770 JF	433	#undef ADDR
1c54d770 JF	434
d66462f5 AH	435	#ifdef __KERNEL__
	436
	437	#include <asm-generic/bitops/sched.h>
	438
	439	#define ARCH_HAS_FAST_MULTIPLIER 1
	440
	441	#include <asm-generic/bitops/hweight.h>
1c54d770	442
d66462f5 AH	443	#endif /* __KERNEL__ */
	444
	445	#include <asm-generic/bitops/fls64.h>
	446
	447	#ifdef __KERNEL__
	448
	449	#include <asm-generic/bitops/ext2-non-atomic.h>
	450
	451	#define ext2_set_bit_atomic(lock, nr, addr) \
	452	test_and_set_bit((nr), (unsigned long *)(addr))
	453	#define ext2_clear_bit_atomic(lock, nr, addr) \
	454	test_and_clear_bit((nr), (unsigned long *)(addr))
	455
	456	#include <asm-generic/bitops/minix.h>
	457
	458	#endif /* __KERNEL__ */
1965aae3	459	#endif /* _ASM_X86_BITOPS_H */