[deliverable/linux.git] / include / asm-mips / bitops.h

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07  Ralf Baechle (ralf@linux-mips.org)
 * Copyright (c) 1999, 2000  Silicon Graphics, Inc.
 */
#ifndef _ASM_BITOPS_H
#define _ASM_BITOPS_H

#include <linux/compiler.h>
#include <linux/irqflags.h>
#include <linux/types.h>
#include <asm/barrier.h>
#include <asm/bug.h>
#include <asm/byteorder.h>		/* sigh ... */
#include <asm/cpu-features.h>
#include <asm/sgidefs.h>
#include <asm/war.h>

#if (_MIPS_SZLONG == 32)
#define SZLONG_LOG 5
#define SZLONG_MASK 31UL
#define __LL		"ll	"
#define __SC		"sc	"
#define __INS		"ins    "
#define __EXT		"ext    "
#elif (_MIPS_SZLONG == 64)
#define SZLONG_LOG 6
#define SZLONG_MASK 63UL
#define __LL		"lld	"
#define __SC		"scd	"
#define __INS		"dins    "
#define __EXT		"dext    "
#endif

/*
 * clear_bit() doesn't provide any barrier for the compiler.
 */
#define smp_mb__before_clear_bit()	smp_mb()
#define smp_mb__after_clear_bit()	smp_mb()

/*
 * set_bit - Atomically set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * This function is atomic and may not be reordered.  See __set_bit()
 * if you do not require the atomic guarantees.
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
{
	unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
	unsigned short bit = nr & SZLONG_MASK;
	unsigned long temp;

	if (cpu_has_llsc && R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	mips3					\n"
		"1:	" __LL "%0, %1			# set_bit	\n"
		"	or	%0, %2					\n"
		"	" __SC	"%0, %1					\n"
		"	beqzl	%0, 1b					\n"
		"	.set	mips0					\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (1UL << bit), "m" (*m));
#ifdef CONFIG_CPU_MIPSR2
	} else if (__builtin_constant_p(bit)) {
		__asm__ __volatile__(
		"1:	" __LL "%0, %1			# set_bit	\n"
		"	" __INS "%0, %4, %2, 1				\n"
		"	" __SC "%0, %1					\n"
		"	beqz	%0, 2f					\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	.previous					\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (bit), "m" (*m), "r" (~0));
#endif /* CONFIG_CPU_MIPSR2 */
	} else if (cpu_has_llsc) {
		__asm__ __volatile__(
		"	.set	mips3					\n"
		"1:	" __LL "%0, %1			# set_bit	\n"
		"	or	%0, %2					\n"
		"	" __SC	"%0, %1					\n"
		"	beqz	%0, 2f					\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	.previous					\n"
		"	.set	mips0					\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (1UL << bit), "m" (*m));
	} else {
		volatile unsigned long *a = addr;
		unsigned long mask;
		unsigned long flags;

		a += nr >> SZLONG_LOG;
		mask = 1UL << bit;
		raw_local_irq_save(flags);
		*a |= mask;
		raw_local_irq_restore(flags);
	}
}

/*
 * clear_bit - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * clear_bit() is atomic and may not be reordered.  However, it does
 * not contain a memory barrier, so if it is used for locking purposes,
 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
 * in order to ensure changes are visible on other processors.
 */
static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
{
	unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
	unsigned short bit = nr & SZLONG_MASK;
	unsigned long temp;

	if (cpu_has_llsc && R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	mips3					\n"
		"1:	" __LL "%0, %1			# clear_bit	\n"
		"	and	%0, %2					\n"
		"	" __SC "%0, %1					\n"
		"	beqzl	%0, 1b					\n"
		"	.set	mips0					\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (~(1UL << bit)), "m" (*m));
#ifdef CONFIG_CPU_MIPSR2
	} else if (__builtin_constant_p(bit)) {
		__asm__ __volatile__(
		"1:	" __LL "%0, %1			# clear_bit	\n"
		"	" __INS "%0, $0, %2, 1				\n"
		"	" __SC "%0, %1					\n"
		"	beqz	%0, 2f					\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	.previous					\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (bit), "m" (*m));
#endif /* CONFIG_CPU_MIPSR2 */
	} else if (cpu_has_llsc) {
		__asm__ __volatile__(
		"	.set	mips3					\n"
		"1:	" __LL "%0, %1			# clear_bit	\n"
		"	and	%0, %2					\n"
		"	" __SC "%0, %1					\n"
		"	beqz	%0, 2f					\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	.previous					\n"
		"	.set	mips0					\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (~(1UL << bit)), "m" (*m));
	} else {
		volatile unsigned long *a = addr;
		unsigned long mask;
		unsigned long flags;

		a += nr >> SZLONG_LOG;
		mask = 1UL << bit;
		raw_local_irq_save(flags);
		*a &= ~mask;
		raw_local_irq_restore(flags);
	}
}

/*
 * change_bit - Toggle a bit in memory
 * @nr: Bit to change
 * @addr: Address to start counting from
 *
 * change_bit() is atomic and may not be reordered.
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
{
	unsigned short bit = nr & SZLONG_MASK;

	if (cpu_has_llsc && R10000_LLSC_WAR) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp;

		__asm__ __volatile__(
		"	.set	mips3				\n"
		"1:	" __LL "%0, %1		# change_bit	\n"
		"	xor	%0, %2				\n"
		"	" __SC	"%0, %1				\n"
		"	beqzl	%0, 1b				\n"
		"	.set	mips0				\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (1UL << bit), "m" (*m));
	} else if (cpu_has_llsc) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp;

		__asm__ __volatile__(
		"	.set	mips3				\n"
		"1:	" __LL "%0, %1		# change_bit	\n"
		"	xor	%0, %2				\n"
		"	" __SC	"%0, %1				\n"
		"	beqz	%0, 2f				\n"
		"	.subsection 2				\n"
		"2:	b	1b				\n"
		"	.previous				\n"
		"	.set	mips0				\n"
		: "=&r" (temp), "=m" (*m)
		: "ir" (1UL << bit), "m" (*m));
	} else {
		volatile unsigned long *a = addr;
		unsigned long mask;
		unsigned long flags;

		a += nr >> SZLONG_LOG;
		mask = 1UL << bit;
		raw_local_irq_save(flags);
		*a ^= mask;
		raw_local_irq_restore(flags);
	}
}

/*
 * test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static inline int test_and_set_bit(unsigned long nr,
	volatile unsigned long *addr)
{
	unsigned short bit = nr & SZLONG_MASK;

	if (cpu_has_llsc && R10000_LLSC_WAR) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp, res;

		__asm__ __volatile__(
		"	.set	mips3					\n"
		"1:	" __LL "%0, %1		# test_and_set_bit	\n"
		"	or	%2, %0, %3				\n"
		"	" __SC	"%2, %1					\n"
		"	beqzl	%2, 1b					\n"
		"	and	%2, %0, %3				\n"
		"	.set	mips0					\n"
		: "=&r" (temp), "=m" (*m), "=&r" (res)
		: "r" (1UL << bit), "m" (*m)
		: "memory");

		return res != 0;
	} else if (cpu_has_llsc) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp, res;

		__asm__ __volatile__(
		"	.set	push					\n"
		"	.set	noreorder				\n"
		"	.set	mips3					\n"
		"1:	" __LL "%0, %1		# test_and_set_bit	\n"
		"	or	%2, %0, %3				\n"
		"	" __SC	"%2, %1					\n"
		"	beqz	%2, 2f					\n"
		"	 and	%2, %0, %3				\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	 nop						\n"
		"	.previous					\n"
		"	.set	pop					\n"
		: "=&r" (temp), "=m" (*m), "=&r" (res)
		: "r" (1UL << bit), "m" (*m)
		: "memory");

		return res != 0;
	} else {
		volatile unsigned long *a = addr;
		unsigned long mask;
		int retval;
		unsigned long flags;

		a += nr >> SZLONG_LOG;
		mask = 1UL << bit;
		raw_local_irq_save(flags);
		retval = (mask & *a) != 0;
		*a |= mask;
		raw_local_irq_restore(flags);

		return retval;
	}

	smp_mb();
}

/*
 * test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static inline int test_and_clear_bit(unsigned long nr,
	volatile unsigned long *addr)
{
	unsigned short bit = nr & SZLONG_MASK;

	if (cpu_has_llsc && R10000_LLSC_WAR) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp, res;

		__asm__ __volatile__(
		"	.set	mips3					\n"
		"1:	" __LL	"%0, %1		# test_and_clear_bit	\n"
		"	or	%2, %0, %3				\n"
		"	xor	%2, %3					\n"
		"	" __SC 	"%2, %1					\n"
		"	beqzl	%2, 1b					\n"
		"	and	%2, %0, %3				\n"
		"	.set	mips0					\n"
		: "=&r" (temp), "=m" (*m), "=&r" (res)
		: "r" (1UL << bit), "m" (*m)
		: "memory");

		return res != 0;
#ifdef CONFIG_CPU_MIPSR2
	} else if (__builtin_constant_p(nr)) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp, res;

		__asm__ __volatile__(
		"1:	" __LL	"%0, %1		# test_and_clear_bit	\n"
		"	" __EXT "%2, %0, %3, 1				\n"
		"	" __INS	"%0, $0, %3, 1				\n"
		"	" __SC 	"%0, %1					\n"
		"	beqz	%0, 2f					\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	.previous					\n"
		: "=&r" (temp), "=m" (*m), "=&r" (res)
		: "ri" (bit), "m" (*m)
		: "memory");

		return res;
#endif
	} else if (cpu_has_llsc) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp, res;

		__asm__ __volatile__(
		"	.set	push					\n"
		"	.set	noreorder				\n"
		"	.set	mips3					\n"
		"1:	" __LL	"%0, %1		# test_and_clear_bit	\n"
		"	or	%2, %0, %3				\n"
		"	xor	%2, %3					\n"
		"	" __SC 	"%2, %1					\n"
		"	beqz	%2, 2f					\n"
		"	 and	%2, %0, %3				\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	 nop						\n"
		"	.previous					\n"
		"	.set	pop					\n"
		: "=&r" (temp), "=m" (*m), "=&r" (res)
		: "r" (1UL << bit), "m" (*m)
		: "memory");

		return res != 0;
	} else {
		volatile unsigned long *a = addr;
		unsigned long mask;
		int retval;
		unsigned long flags;

		a += nr >> SZLONG_LOG;
		mask = 1UL << bit;
		raw_local_irq_save(flags);
		retval = (mask & *a) != 0;
		*a &= ~mask;
		raw_local_irq_restore(flags);

		return retval;
	}

	smp_mb();
}

/*
 * test_and_change_bit - Change a bit and return its old value
 * @nr: Bit to change
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static inline int test_and_change_bit(unsigned long nr,
	volatile unsigned long *addr)
{
	unsigned short bit = nr & SZLONG_MASK;

	if (cpu_has_llsc && R10000_LLSC_WAR) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp, res;

		__asm__ __volatile__(
		"	.set	mips3					\n"
		"1:	" __LL	"%0, %1		# test_and_change_bit	\n"
		"	xor	%2, %0, %3				\n"
		"	" __SC	"%2, %1					\n"
		"	beqzl	%2, 1b					\n"
		"	and	%2, %0, %3				\n"
		"	.set	mips0					\n"
		: "=&r" (temp), "=m" (*m), "=&r" (res)
		: "r" (1UL << bit), "m" (*m)
		: "memory");

		return res != 0;
	} else if (cpu_has_llsc) {
		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
		unsigned long temp, res;

		__asm__ __volatile__(
		"	.set	push					\n"
		"	.set	noreorder				\n"
		"	.set	mips3					\n"
		"1:	" __LL	"%0, %1		# test_and_change_bit	\n"
		"	xor	%2, %0, %3				\n"
		"	" __SC	"\t%2, %1				\n"
		"	beqz	%2, 2f					\n"
		"	 and	%2, %0, %3				\n"
		"	.subsection 2					\n"
		"2:	b	1b					\n"
		"	 nop						\n"
		"	.previous					\n"
		"	.set	pop					\n"
		: "=&r" (temp), "=m" (*m), "=&r" (res)
		: "r" (1UL << bit), "m" (*m)
		: "memory");

		return res != 0;
	} else {
		volatile unsigned long *a = addr;
		unsigned long mask, retval;
		unsigned long flags;

		a += nr >> SZLONG_LOG;
		mask = 1UL << bit;
		raw_local_irq_save(flags);
		retval = (mask & *a) != 0;
		*a ^= mask;
		raw_local_irq_restore(flags);

		return retval;
	}

	smp_mb();
}

#include <asm-generic/bitops/non-atomic.h>

/*
 * Return the bit position (0..63) of the most significant 1 bit in a word
 * Returns -1 if no 1 bit exists
 */
static inline int __ilog2(unsigned long x)
{
	int lz;

	if (sizeof(x) == 4) {
		__asm__ (
		"	.set	push					\n"
		"	.set	mips32					\n"
		"	clz	%0, %1					\n"
		"	.set	pop					\n"
		: "=r" (lz)
		: "r" (x));

		return 31 - lz;
	}

	BUG_ON(sizeof(x) != 8);

	__asm__ (
	"	.set	push						\n"
	"	.set	mips64						\n"
	"	dclz	%0, %1						\n"
	"	.set	pop						\n"
	: "=r" (lz)
	: "r" (x));

	return 63 - lz;
}

#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64)

/*
 * __ffs - find first bit in word.
 * @word: The word to search
 *
 * Returns 0..SZLONG-1
 * Undefined if no bit exists, so code should check against 0 first.
 */
static inline unsigned long __ffs(unsigned long word)
{
	return __ilog2(word & -word);
}

/*
 * fls - find last bit set.
 * @word: The word to search
 *
 * This is defined the same way as ffs.
 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 */
static inline int fls(int word)
{
	__asm__ ("clz %0, %1" : "=r" (word) : "r" (word));

	return 32 - word;
}

#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPS64)
static inline int fls64(__u64 word)
{
	__asm__ ("dclz %0, %1" : "=r" (word) : "r" (word));

	return 64 - word;
}
#else
#include <asm-generic/bitops/fls64.h>
#endif

/*
 * ffs - find first bit set.
 * @word: The word to search
 *
 * This is defined the same way as
 * the libc and compiler builtin ffs routines, therefore
 * differs in spirit from the above ffz (man ffs).
 */
static inline int ffs(int word)
{
	if (!word)
		return 0;

	return fls(word & -word);
}

#else

#include <asm-generic/bitops/__ffs.h>
#include <asm-generic/bitops/ffs.h>
#include <asm-generic/bitops/fls.h>
#include <asm-generic/bitops/fls64.h>

#endif /*defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) */

#include <asm-generic/bitops/ffz.h>
#include <asm-generic/bitops/find.h>

#ifdef __KERNEL__

#include <asm-generic/bitops/sched.h>
#include <asm-generic/bitops/hweight.h>
#include <asm-generic/bitops/ext2-non-atomic.h>
#include <asm-generic/bitops/ext2-atomic.h>
#include <asm-generic/bitops/minix.h>

#endif /* __KERNEL__ */

#endif /* _ASM_BITOPS_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
102fa15c	6	* Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
1da177e4 LT	7	* Copyright (c) 1999, 2000 Silicon Graphics, Inc.
	8	*/
	9	#ifndef _ASM_BITOPS_H
	10	#define _ASM_BITOPS_H
	11
1da177e4	12	#include <linux/compiler.h>
4ffd8b38	13	#include <linux/irqflags.h>
1da177e4	14	#include <linux/types.h>
0004a9df	15	#include <asm/barrier.h>
ec917c2c	16	#include <asm/bug.h>
1da177e4 LT	17	#include <asm/byteorder.h> /* sigh ... */
1da177e4 LT	18	#include <asm/cpu-features.h>
4ffd8b38 RB	19	#include <asm/sgidefs.h>
4ffd8b38 RB	20	#include <asm/war.h>
1da177e4 LT	21
	22	#if (_MIPS_SZLONG == 32)
	23	#define SZLONG_LOG 5
	24	#define SZLONG_MASK 31UL
aac8aa77 MR	25	#define __LL "ll "
aac8aa77 MR	26	#define __SC "sc "
102fa15c RB	27	#define __INS "ins "
102fa15c RB	28	#define __EXT "ext "
1da177e4 LT	29	#elif (_MIPS_SZLONG == 64)
	30	#define SZLONG_LOG 6
	31	#define SZLONG_MASK 63UL
aac8aa77 MR	32	#define __LL "lld "
aac8aa77 MR	33	#define __SC "scd "
102fa15c RB	34	#define __INS "dins "
102fa15c RB	35	#define __EXT "dext "
1da177e4 LT	36	#endif
1da177e4 LT	37
1da177e4 LT	38	/*
	39	* clear_bit() doesn't provide any barrier for the compiler.
	40	*/
	41	#define smp_mb__before_clear_bit() smp_mb()
	42	#define smp_mb__after_clear_bit() smp_mb()
	43
1da177e4 LT	44	/*
	45	* set_bit - Atomically set a bit in memory
	46	* @nr: the bit to set
	47	* @addr: the address to start counting from
	48	*
	49	* This function is atomic and may not be reordered. See __set_bit()
	50	* if you do not require the atomic guarantees.
	51	* Note that @nr may be almost arbitrarily large; this function is not
	52	* restricted to acting on a single-word quantity.
	53	*/
	54	static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
	55	{
	56	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
b961153b	57	unsigned short bit = nr & SZLONG_MASK;
1da177e4 LT	58	unsigned long temp;
	59
	60	if (cpu_has_llsc && R10000_LLSC_WAR) {
	61	__asm__ __volatile__(
c4559f67	62	" .set mips3 \n"
1da177e4 LT	63	"1: " __LL "%0, %1 # set_bit \n"
1da177e4 LT	64	" or %0, %2 \n"
aac8aa77	65	" " __SC "%0, %1 \n"
1da177e4	66	" beqzl %0, 1b \n"
aac8aa77	67	" .set mips0 \n"
1da177e4	68	: "=&r" (temp), "=m" (*m)
b961153b	69	: "ir" (1UL << bit), "m" (*m));
102fa15c	70	#ifdef CONFIG_CPU_MIPSR2
b961153b	71	} else if (__builtin_constant_p(bit)) {
102fa15c RB	72	__asm__ __volatile__(
	73	"1: " __LL "%0, %1 # set_bit \n"
	74	" " __INS "%0, %4, %2, 1 \n"
	75	" " __SC "%0, %1 \n"
	76	" beqz %0, 2f \n"
	77	" .subsection 2 \n"
	78	"2: b 1b \n"
	79	" .previous \n"
	80	: "=&r" (temp), "=m" (*m)
b961153b	81	: "ir" (bit), "m" (*m), "r" (~0));
102fa15c	82	#endif /* CONFIG_CPU_MIPSR2 */
1da177e4 LT	83	} else if (cpu_has_llsc) {
1da177e4 LT	84	__asm__ __volatile__(
c4559f67	85	" .set mips3 \n"
1da177e4 LT	86	"1: " __LL "%0, %1 # set_bit \n"
1da177e4 LT	87	" or %0, %2 \n"
aac8aa77	88	" " __SC "%0, %1 \n"
f65e4fa8 RB	89	" beqz %0, 2f \n"
	90	" .subsection 2 \n"
	91	"2: b 1b \n"
	92	" .previous \n"
aac8aa77	93	" .set mips0 \n"
1da177e4	94	: "=&r" (temp), "=m" (*m)
b961153b	95	: "ir" (1UL << bit), "m" (*m));
1da177e4 LT	96	} else {
	97	volatile unsigned long *a = addr;
	98	unsigned long mask;
4ffd8b38	99	unsigned long flags;
1da177e4 LT	100
1da177e4 LT	101	a += nr >> SZLONG_LOG;
b961153b	102	mask = 1UL << bit;
49edd098	103	raw_local_irq_save(flags);
1da177e4	104	*a \|= mask;
49edd098	105	raw_local_irq_restore(flags);
1da177e4 LT	106	}
	107	}
	108
1da177e4 LT	109	/*
	110	* clear_bit - Clears a bit in memory
	111	* @nr: Bit to clear
	112	* @addr: Address to start counting from
	113	*
	114	* clear_bit() is atomic and may not be reordered. However, it does
	115	* not contain a memory barrier, so if it is used for locking purposes,
	116	* you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
	117	* in order to ensure changes are visible on other processors.
	118	*/
	119	static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
	120	{
	121	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
b961153b	122	unsigned short bit = nr & SZLONG_MASK;
1da177e4 LT	123	unsigned long temp;
	124
	125	if (cpu_has_llsc && R10000_LLSC_WAR) {
	126	__asm__ __volatile__(
c4559f67	127	" .set mips3 \n"
1da177e4 LT	128	"1: " __LL "%0, %1 # clear_bit \n"
	129	" and %0, %2 \n"
	130	" " __SC "%0, %1 \n"
	131	" beqzl %0, 1b \n"
aac8aa77	132	" .set mips0 \n"
1da177e4	133	: "=&r" (temp), "=m" (*m)
b961153b	134	: "ir" (~(1UL << bit)), "m" (*m));
102fa15c	135	#ifdef CONFIG_CPU_MIPSR2
b961153b	136	} else if (__builtin_constant_p(bit)) {
102fa15c RB	137	__asm__ __volatile__(
	138	"1: " __LL "%0, %1 # clear_bit \n"
	139	" " __INS "%0, $0, %2, 1 \n"
	140	" " __SC "%0, %1 \n"
	141	" beqz %0, 2f \n"
	142	" .subsection 2 \n"
	143	"2: b 1b \n"
	144	" .previous \n"
	145	: "=&r" (temp), "=m" (*m)
b961153b	146	: "ir" (bit), "m" (*m));
102fa15c	147	#endif /* CONFIG_CPU_MIPSR2 */
1da177e4 LT	148	} else if (cpu_has_llsc) {
1da177e4 LT	149	__asm__ __volatile__(
c4559f67	150	" .set mips3 \n"
1da177e4 LT	151	"1: " __LL "%0, %1 # clear_bit \n"
	152	" and %0, %2 \n"
	153	" " __SC "%0, %1 \n"
f65e4fa8 RB	154	" beqz %0, 2f \n"
	155	" .subsection 2 \n"
	156	"2: b 1b \n"
	157	" .previous \n"
aac8aa77	158	" .set mips0 \n"
1da177e4	159	: "=&r" (temp), "=m" (*m)
b961153b	160	: "ir" (~(1UL << bit)), "m" (*m));
1da177e4 LT	161	} else {
	162	volatile unsigned long *a = addr;
	163	unsigned long mask;
4ffd8b38	164	unsigned long flags;
1da177e4 LT	165
1da177e4 LT	166	a += nr >> SZLONG_LOG;
b961153b	167	mask = 1UL << bit;
49edd098	168	raw_local_irq_save(flags);
1da177e4	169	*a &= ~mask;
49edd098	170	raw_local_irq_restore(flags);
1da177e4 LT	171	}
	172	}
	173
1da177e4 LT	174	/*
	175	* change_bit - Toggle a bit in memory
	176	* @nr: Bit to change
	177	* @addr: Address to start counting from
	178	*
	179	* change_bit() is atomic and may not be reordered.
	180	* Note that @nr may be almost arbitrarily large; this function is not
	181	* restricted to acting on a single-word quantity.
	182	*/
	183	static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
	184	{
b961153b RB	185	unsigned short bit = nr & SZLONG_MASK;
b961153b RB	186
1da177e4 LT	187	if (cpu_has_llsc && R10000_LLSC_WAR) {
	188	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	189	unsigned long temp;
	190
	191	__asm__ __volatile__(
c4559f67	192	" .set mips3 \n"
1da177e4 LT	193	"1: " __LL "%0, %1 # change_bit \n"
1da177e4 LT	194	" xor %0, %2 \n"
aac8aa77	195	" " __SC "%0, %1 \n"
1da177e4	196	" beqzl %0, 1b \n"
aac8aa77	197	" .set mips0 \n"
1da177e4	198	: "=&r" (temp), "=m" (*m)
b961153b	199	: "ir" (1UL << bit), "m" (*m));
1da177e4 LT	200	} else if (cpu_has_llsc) {
	201	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	202	unsigned long temp;
	203
	204	__asm__ __volatile__(
c4559f67	205	" .set mips3 \n"
1da177e4 LT	206	"1: " __LL "%0, %1 # change_bit \n"
1da177e4 LT	207	" xor %0, %2 \n"
aac8aa77	208	" " __SC "%0, %1 \n"
f65e4fa8 RB	209	" beqz %0, 2f \n"
	210	" .subsection 2 \n"
	211	"2: b 1b \n"
	212	" .previous \n"
aac8aa77	213	" .set mips0 \n"
1da177e4	214	: "=&r" (temp), "=m" (*m)
b961153b	215	: "ir" (1UL << bit), "m" (*m));
1da177e4 LT	216	} else {
	217	volatile unsigned long *a = addr;
	218	unsigned long mask;
4ffd8b38	219	unsigned long flags;
1da177e4 LT	220
1da177e4 LT	221	a += nr >> SZLONG_LOG;
b961153b	222	mask = 1UL << bit;
49edd098	223	raw_local_irq_save(flags);
1da177e4	224	*a ^= mask;
49edd098	225	raw_local_irq_restore(flags);
1da177e4 LT	226	}
	227	}
	228
1da177e4 LT	229	/*
	230	* test_and_set_bit - Set a bit and return its old value
	231	* @nr: Bit to set
	232	* @addr: Address to count from
	233	*
	234	* This operation is atomic and cannot be reordered.
	235	* It also implies a memory barrier.
	236	*/
	237	static inline int test_and_set_bit(unsigned long nr,
	238	volatile unsigned long *addr)
	239	{
b961153b RB	240	unsigned short bit = nr & SZLONG_MASK;
b961153b RB	241
1da177e4 LT	242	if (cpu_has_llsc && R10000_LLSC_WAR) {
	243	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	244	unsigned long temp, res;
	245
	246	__asm__ __volatile__(
c4559f67	247	" .set mips3 \n"
1da177e4 LT	248	"1: " __LL "%0, %1 # test_and_set_bit \n"
	249	" or %2, %0, %3 \n"
	250	" " __SC "%2, %1 \n"
	251	" beqzl %2, 1b \n"
	252	" and %2, %0, %3 \n"
aac8aa77	253	" .set mips0 \n"
1da177e4	254	: "=&r" (temp), "=m" (*m), "=&r" (res)
b961153b	255	: "r" (1UL << bit), "m" (*m)
1da177e4 LT	256	: "memory");
	257
	258	return res != 0;
	259	} else if (cpu_has_llsc) {
	260	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	261	unsigned long temp, res;
	262
	263	__asm__ __volatile__(
aac8aa77 MR	264	" .set push \n"
aac8aa77 MR	265	" .set noreorder \n"
c4559f67	266	" .set mips3 \n"
aac8aa77	267	"1: " __LL "%0, %1 # test_and_set_bit \n"
1da177e4 LT	268	" or %2, %0, %3 \n"
1da177e4 LT	269	" " __SC "%2, %1 \n"
f65e4fa8	270	" beqz %2, 2f \n"
1da177e4	271	" and %2, %0, %3 \n"
f65e4fa8 RB	272	" .subsection 2 \n"
	273	"2: b 1b \n"
	274	" nop \n"
	275	" .previous \n"
aac8aa77	276	" .set pop \n"
1da177e4	277	: "=&r" (temp), "=m" (*m), "=&r" (res)
b961153b	278	: "r" (1UL << bit), "m" (*m)
1da177e4 LT	279	: "memory");
	280
	281	return res != 0;
	282	} else {
	283	volatile unsigned long *a = addr;
	284	unsigned long mask;
	285	int retval;
4ffd8b38	286	unsigned long flags;
1da177e4 LT	287
1da177e4 LT	288	a += nr >> SZLONG_LOG;
b961153b	289	mask = 1UL << bit;
49edd098	290	raw_local_irq_save(flags);
1da177e4 LT	291	retval = (mask & *a) != 0;
1da177e4 LT	292	*a \|= mask;
49edd098	293	raw_local_irq_restore(flags);
1da177e4 LT	294
	295	return retval;
	296	}
0004a9df RB	297
0004a9df RB	298	smp_mb();
1da177e4 LT	299	}
1da177e4 LT	300
1da177e4 LT	301	/*
	302	* test_and_clear_bit - Clear a bit and return its old value
	303	* @nr: Bit to clear
	304	* @addr: Address to count from
	305	*
	306	* This operation is atomic and cannot be reordered.
	307	* It also implies a memory barrier.
	308	*/
	309	static inline int test_and_clear_bit(unsigned long nr,
	310	volatile unsigned long *addr)
	311	{
b961153b RB	312	unsigned short bit = nr & SZLONG_MASK;
b961153b RB	313
1da177e4 LT	314	if (cpu_has_llsc && R10000_LLSC_WAR) {
	315	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	316	unsigned long temp, res;
	317
	318	__asm__ __volatile__(
c4559f67	319	" .set mips3 \n"
1da177e4 LT	320	"1: " __LL "%0, %1 # test_and_clear_bit \n"
	321	" or %2, %0, %3 \n"
	322	" xor %2, %3 \n"
aac8aa77	323	" " __SC "%2, %1 \n"
1da177e4 LT	324	" beqzl %2, 1b \n"
1da177e4 LT	325	" and %2, %0, %3 \n"
aac8aa77	326	" .set mips0 \n"
1da177e4	327	: "=&r" (temp), "=m" (*m), "=&r" (res)
b961153b	328	: "r" (1UL << bit), "m" (*m)
1da177e4 LT	329	: "memory");
	330
	331	return res != 0;
102fa15c RB	332	#ifdef CONFIG_CPU_MIPSR2
	333	} else if (__builtin_constant_p(nr)) {
	334	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	335	unsigned long temp, res;
	336
	337	__asm__ __volatile__(
	338	"1: " __LL "%0, %1 # test_and_clear_bit \n"
	339	" " __EXT "%2, %0, %3, 1 \n"
	340	" " __INS "%0, $0, %3, 1 \n"
	341	" " __SC "%0, %1 \n"
	342	" beqz %0, 2f \n"
	343	" .subsection 2 \n"
	344	"2: b 1b \n"
	345	" .previous \n"
	346	: "=&r" (temp), "=m" (*m), "=&r" (res)
b961153b	347	: "ri" (bit), "m" (*m)
102fa15c RB	348	: "memory");
	349
	350	return res;
	351	#endif
1da177e4 LT	352	} else if (cpu_has_llsc) {
	353	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	354	unsigned long temp, res;
	355
	356	__asm__ __volatile__(
aac8aa77 MR	357	" .set push \n"
aac8aa77 MR	358	" .set noreorder \n"
c4559f67	359	" .set mips3 \n"
aac8aa77	360	"1: " __LL "%0, %1 # test_and_clear_bit \n"
1da177e4 LT	361	" or %2, %0, %3 \n"
1da177e4 LT	362	" xor %2, %3 \n"
aac8aa77	363	" " __SC "%2, %1 \n"
f65e4fa8	364	" beqz %2, 2f \n"
1da177e4	365	" and %2, %0, %3 \n"
f65e4fa8 RB	366	" .subsection 2 \n"
	367	"2: b 1b \n"
	368	" nop \n"
	369	" .previous \n"
aac8aa77	370	" .set pop \n"
1da177e4	371	: "=&r" (temp), "=m" (*m), "=&r" (res)
b961153b	372	: "r" (1UL << bit), "m" (*m)
1da177e4 LT	373	: "memory");
	374
	375	return res != 0;
	376	} else {
	377	volatile unsigned long *a = addr;
	378	unsigned long mask;
	379	int retval;
4ffd8b38	380	unsigned long flags;
1da177e4 LT	381
1da177e4 LT	382	a += nr >> SZLONG_LOG;
b961153b	383	mask = 1UL << bit;
49edd098	384	raw_local_irq_save(flags);
1da177e4 LT	385	retval = (mask & *a) != 0;
1da177e4 LT	386	*a &= ~mask;
49edd098	387	raw_local_irq_restore(flags);
1da177e4 LT	388
	389	return retval;
	390	}
0004a9df RB	391
0004a9df RB	392	smp_mb();
1da177e4 LT	393	}
1da177e4 LT	394
1da177e4 LT	395	/*
	396	* test_and_change_bit - Change a bit and return its old value
	397	* @nr: Bit to change
	398	* @addr: Address to count from
	399	*
	400	* This operation is atomic and cannot be reordered.
	401	* It also implies a memory barrier.
	402	*/
	403	static inline int test_and_change_bit(unsigned long nr,
	404	volatile unsigned long *addr)
	405	{
b961153b RB	406	unsigned short bit = nr & SZLONG_MASK;
b961153b RB	407
1da177e4 LT	408	if (cpu_has_llsc && R10000_LLSC_WAR) {
	409	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	410	unsigned long temp, res;
	411
	412	__asm__ __volatile__(
c4559f67	413	" .set mips3 \n"
aac8aa77	414	"1: " __LL "%0, %1 # test_and_change_bit \n"
1da177e4	415	" xor %2, %0, %3 \n"
aac8aa77	416	" " __SC "%2, %1 \n"
1da177e4 LT	417	" beqzl %2, 1b \n"
1da177e4 LT	418	" and %2, %0, %3 \n"
aac8aa77	419	" .set mips0 \n"
1da177e4	420	: "=&r" (temp), "=m" (*m), "=&r" (res)
b961153b	421	: "r" (1UL << bit), "m" (*m)
1da177e4 LT	422	: "memory");
	423
	424	return res != 0;
	425	} else if (cpu_has_llsc) {
	426	unsigned long m = ((unsigned long ) addr) + (nr >> SZLONG_LOG);
	427	unsigned long temp, res;
	428
	429	__asm__ __volatile__(
aac8aa77 MR	430	" .set push \n"
aac8aa77 MR	431	" .set noreorder \n"
c4559f67	432	" .set mips3 \n"
aac8aa77	433	"1: " __LL "%0, %1 # test_and_change_bit \n"
1da177e4	434	" xor %2, %0, %3 \n"
aac8aa77	435	" " __SC "\t%2, %1 \n"
f65e4fa8	436	" beqz %2, 2f \n"
1da177e4	437	" and %2, %0, %3 \n"
f65e4fa8 RB	438	" .subsection 2 \n"
	439	"2: b 1b \n"
	440	" nop \n"
	441	" .previous \n"
aac8aa77	442	" .set pop \n"
1da177e4	443	: "=&r" (temp), "=m" (*m), "=&r" (res)
b961153b	444	: "r" (1UL << bit), "m" (*m)
1da177e4 LT	445	: "memory");
	446
	447	return res != 0;
	448	} else {
	449	volatile unsigned long *a = addr;
	450	unsigned long mask, retval;
4ffd8b38	451	unsigned long flags;
1da177e4 LT	452
1da177e4 LT	453	a += nr >> SZLONG_LOG;
b961153b	454	mask = 1UL << bit;
49edd098	455	raw_local_irq_save(flags);
1da177e4 LT	456	retval = (mask & *a) != 0;
1da177e4 LT	457	*a ^= mask;
49edd098	458	raw_local_irq_restore(flags);
1da177e4 LT	459
	460	return retval;
	461	}
0004a9df RB	462
0004a9df RB	463	smp_mb();
1da177e4 LT	464	}
1da177e4 LT	465
3c9ee7ef	466	#include <asm-generic/bitops/non-atomic.h>
1da177e4 LT	467
1da177e4 LT	468	/*
ec917c2c	469	* Return the bit position (0..63) of the most significant 1 bit in a word
65903265 RB	470	* Returns -1 if no 1 bit exists
65903265 RB	471	*/
ec917c2c	472	static inline int __ilog2(unsigned long x)
65903265 RB	473	{
	474	int lz;
	475
ec917c2c RB	476	if (sizeof(x) == 4) {
	477	__asm__ (
	478	" .set push \n"
	479	" .set mips32 \n"
	480	" clz %0, %1 \n"
	481	" .set pop \n"
	482	: "=r" (lz)
	483	: "r" (x));
65903265	484
ec917c2c RB	485	return 31 - lz;
	486	}
	487
	488	BUG_ON(sizeof(x) != 8);
65903265 RB	489
	490	__asm__ (
	491	" .set push \n"
	492	" .set mips64 \n"
	493	" dclz %0, %1 \n"
	494	" .set pop \n"
	495	: "=r" (lz)
	496	: "r" (x));
	497
	498	return 63 - lz;
	499	}
65903265	500
3c9ee7ef AM	501	#if defined(CONFIG_CPU_MIPS32) \|\| defined(CONFIG_CPU_MIPS64)
3c9ee7ef AM	502
65903265 RB	503	/*
65903265 RB	504	* __ffs - find first bit in word.
1da177e4 LT	505	* @word: The word to search
1da177e4 LT	506	*
65903265 RB	507	* Returns 0..SZLONG-1
65903265 RB	508	* Undefined if no bit exists, so code should check against 0 first.
1da177e4	509	*/
65903265	510	static inline unsigned long __ffs(unsigned long word)
1da177e4	511	{
65903265	512	return __ilog2(word & -word);
1da177e4 LT	513	}
	514
	515	/*
bc818247	516	* fls - find last bit set.
1da177e4 LT	517	* @word: The word to search
1da177e4 LT	518	*
bc818247 AN	519	* This is defined the same way as ffs.
bc818247 AN	520	* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
1da177e4	521	*/
bc818247	522	static inline int fls(int word)
1da177e4	523	{
bc818247	524	__asm__ ("clz %0, %1" : "=r" (word) : "r" (word));
65903265	525
bc818247	526	return 32 - word;
1da177e4 LT	527	}
1da177e4 LT	528
bc818247 AN	529	#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPS64)
bc818247 AN	530	static inline int fls64(__u64 word)
65903265	531	{
bc818247 AN	532	__asm__ ("dclz %0, %1" : "=r" (word) : "r" (word));
	533
	534	return 64 - word;
65903265	535	}
bc818247 AN	536	#else
	537	#include <asm-generic/bitops/fls64.h>
	538	#endif
65903265 RB	539
65903265 RB	540	/*
bc818247	541	* ffs - find first bit set.
65903265 RB	542	* @word: The word to search
65903265 RB	543	*
bc818247 AN	544	* This is defined the same way as
	545	* the libc and compiler builtin ffs routines, therefore
	546	* differs in spirit from the above ffz (man ffs).
65903265	547	*/
bc818247	548	static inline int ffs(int word)
65903265	549	{
bc818247 AN	550	if (!word)
bc818247 AN	551	return 0;
2caf1900	552
bc818247	553	return fls(word & -word);
65903265 RB	554	}
65903265 RB	555
3c9ee7ef	556	#else
1da177e4	557
3c9ee7ef AM	558	#include <asm-generic/bitops/__ffs.h>
3c9ee7ef AM	559	#include <asm-generic/bitops/ffs.h>
3c9ee7ef	560	#include <asm-generic/bitops/fls.h>
bc818247	561	#include <asm-generic/bitops/fls64.h>
1da177e4	562
3c9ee7ef	563	#endif /defined(CONFIG_CPU_MIPS32) \|\| defined(CONFIG_CPU_MIPS64) /
1da177e4	564
bc818247	565	#include <asm-generic/bitops/ffz.h>
3c9ee7ef	566	#include <asm-generic/bitops/find.h>
1da177e4 LT	567
	568	#ifdef __KERNEL__
	569
3c9ee7ef AM	570	#include <asm-generic/bitops/sched.h>
	571	#include <asm-generic/bitops/hweight.h>
	572	#include <asm-generic/bitops/ext2-non-atomic.h>
	573	#include <asm-generic/bitops/ext2-atomic.h>
	574	#include <asm-generic/bitops/minix.h>
1da177e4 LT	575
	576	#endif /* __KERNEL__ */
	577
	578	#endif /* _ASM_BITOPS_H */