[deliverable/linux.git] / kernel / spinlock.c

/*
 * Copyright (2004) Linus Torvalds
 *
 * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
 *
 * Copyright (2004, 2005) Ingo Molnar
 *
 * This file contains the spinlock/rwlock implementations for the
 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
 *
 * Note that some architectures have special knowledge about the
 * stack frames of these functions in their profile_pc. If you
 * change anything significant here that could change the stack
 * frame contact the architecture maintainers.
 */

#include <linux/linkage.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/debug_locks.h>
#include <linux/module.h>

int __lockfunc _spin_trylock(spinlock_t *lock)
{
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}
	
	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_spin_trylock);

int __lockfunc _read_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_read_trylock(lock)) {
		rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_read_trylock);

int __lockfunc _write_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_write_trylock(lock)) {
		rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_write_trylock);

/*
 * If lockdep is enabled then we use the non-preemption spin-ops
 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
 */
#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)

void __lockfunc _read_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
}
EXPORT_SYMBOL(_read_lock);

unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	/*
	 * On lockdep we dont want the hand-coded irq-enable of
	 * _raw_spin_lock_flags() code, because lockdep assumes
	 * that interrupts are not re-enabled during lock-acquire:
	 */
#ifdef CONFIG_LOCKDEP
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
#else
	_raw_spin_lock_flags(lock, &flags);
#endif
	return flags;
}
EXPORT_SYMBOL(_spin_lock_irqsave);

void __lockfunc _spin_lock_irq(spinlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_irq);

void __lockfunc _spin_lock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_bh);

unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
	return flags;
}
EXPORT_SYMBOL(_read_lock_irqsave);

void __lockfunc _read_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
}
EXPORT_SYMBOL(_read_lock_irq);

void __lockfunc _read_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
}
EXPORT_SYMBOL(_read_lock_bh);

unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
	return flags;
}
EXPORT_SYMBOL(_write_lock_irqsave);

void __lockfunc _write_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
}
EXPORT_SYMBOL(_write_lock_irq);

void __lockfunc _write_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
}
EXPORT_SYMBOL(_write_lock_bh);

void __lockfunc _spin_lock(spinlock_t *lock)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}

EXPORT_SYMBOL(_spin_lock);

void __lockfunc _write_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
}

EXPORT_SYMBOL(_write_lock);

#else /* CONFIG_PREEMPT: */

/*
 * This could be a long-held lock. We both prepare to spin for a long
 * time (making _this_ CPU preemptable if possible), and we also signal
 * towards that other CPU that it should break the lock ASAP.
 *
 * (We do this in a function because inlining it would be excessive.)
 */

#define BUILD_LOCK_OPS(op, locktype)					\
void __lockfunc _##op##_lock(locktype##_t *lock)			\
{									\
	for (;;) {							\
		preempt_disable();					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			_raw_##op##_relax(&lock->raw_lock);		\
	}								\
	(lock)->break_lock = 0;						\
}									\
									\
EXPORT_SYMBOL(_##op##_lock);						\
									\
unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)	\
{									\
	unsigned long flags;						\
									\
	for (;;) {							\
		preempt_disable();					\
		local_irq_save(flags);					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		local_irq_restore(flags);				\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			_raw_##op##_relax(&lock->raw_lock);		\
	}								\
	(lock)->break_lock = 0;						\
	return flags;							\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irqsave);					\
									\
void __lockfunc _##op##_lock_irq(locktype##_t *lock)			\
{									\
	_##op##_lock_irqsave(lock);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irq);					\
									\
void __lockfunc _##op##_lock_bh(locktype##_t *lock)			\
{									\
	unsigned long flags;						\
									\
	/*							*/	\
	/* Careful: we must exclude softirqs too, hence the	*/	\
	/* irq-disabling. We use the generic preemption-aware	*/	\
	/* function:						*/	\
	/**/								\
	flags = _##op##_lock_irqsave(lock);				\
	local_bh_disable();						\
	local_irq_restore(flags);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_bh)

/*
 * Build preemption-friendly versions of the following
 * lock-spinning functions:
 *
 *         _[spin|read|write]_lock()
 *         _[spin|read|write]_lock_irq()
 *         _[spin|read|write]_lock_irqsave()
 *         _[spin|read|write]_lock_bh()
 */
BUILD_LOCK_OPS(spin, spinlock);
BUILD_LOCK_OPS(read, rwlock);
BUILD_LOCK_OPS(write, rwlock);

#endif /* CONFIG_PREEMPT */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_nested);

unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	/*
	 * On lockdep we dont want the hand-coded irq-enable of
	 * _raw_spin_lock_flags() code, because lockdep assumes
	 * that interrupts are not re-enabled during lock-acquire:
	 */
#ifdef CONFIG_LOCKDEP
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
#else
	_raw_spin_lock_flags(lock, &flags);
#endif
	return flags;
}
EXPORT_SYMBOL(_spin_lock_irqsave_nested);

void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
				     struct lockdep_map *nest_lock)
{
	preempt_disable();
	spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_nest_lock);

#endif

void __lockfunc _spin_unlock(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock);

void __lockfunc _write_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock);

void __lockfunc _read_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock);

void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irqrestore);

void __lockfunc _spin_unlock_irq(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irq);

void __lockfunc _spin_unlock_bh(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_spin_unlock_bh);

void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irqrestore);

void __lockfunc _read_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irq);

void __lockfunc _read_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_read_unlock_bh);

void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irqrestore);

void __lockfunc _write_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irq);

void __lockfunc _write_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_write_unlock_bh);

int __lockfunc _spin_trylock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	return 0;
}
EXPORT_SYMBOL(_spin_trylock_bh);

notrace int in_lock_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __lockfunc functions */
	extern char __lock_text_start[], __lock_text_end[];

	return addr >= (unsigned long)__lock_text_start
	&& addr < (unsigned long)__lock_text_end;
}
EXPORT_SYMBOL(in_lock_functions);
Commit	Line	Data
	1	/*
	2	* Copyright (2004) Linus Torvalds
	3	*
	4	* Author: Zwane Mwaikambo <zwane@fsmlabs.com>
	5	*
	6	* Copyright (2004, 2005) Ingo Molnar
	7	*
	8	* This file contains the spinlock/rwlock implementations for the
	9	* SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
	10	*
	11	* Note that some architectures have special knowledge about the
	12	* stack frames of these functions in their profile_pc. If you
	13	* change anything significant here that could change the stack
	14	* frame contact the architecture maintainers.
	15	*/
	16
	17	#include <linux/linkage.h>
	18	#include <linux/preempt.h>
	19	#include <linux/spinlock.h>
	20	#include <linux/interrupt.h>
	21	#include <linux/debug_locks.h>
	22	#include <linux/module.h>
	23
	24	int __lockfunc _spin_trylock(spinlock_t *lock)
	25	{
	26	preempt_disable();
	27	if (_raw_spin_trylock(lock)) {
	28	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
	29	return 1;
	30	}
	31
	32	preempt_enable();
	33	return 0;
	34	}
	35	EXPORT_SYMBOL(_spin_trylock);
	36
	37	int __lockfunc _read_trylock(rwlock_t *lock)
	38	{
	39	preempt_disable();
	40	if (_raw_read_trylock(lock)) {
	41	rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
	42	return 1;
	43	}
	44
	45	preempt_enable();
	46	return 0;
	47	}
	48	EXPORT_SYMBOL(_read_trylock);
	49
	50	int __lockfunc _write_trylock(rwlock_t *lock)
	51	{
	52	preempt_disable();
	53	if (_raw_write_trylock(lock)) {
	54	rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
	55	return 1;
	56	}
	57
	58	preempt_enable();
	59	return 0;
	60	}
	61	EXPORT_SYMBOL(_write_trylock);
	62
	63	/*
	64	* If lockdep is enabled then we use the non-preemption spin-ops
	65	* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
	66	* not re-enabled during lock-acquire (which the preempt-spin-ops do):
	67	*/
	68	#if !defined(CONFIG_GENERIC_LOCKBREAK) \|\| defined(CONFIG_DEBUG_LOCK_ALLOC)
	69
	70	void __lockfunc _read_lock(rwlock_t *lock)
	71	{
	72	preempt_disable();
	73	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	74	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
	75	}
	76	EXPORT_SYMBOL(_read_lock);
	77
	78	unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
	79	{
	80	unsigned long flags;
	81
	82	local_irq_save(flags);
	83	preempt_disable();
	84	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	85	/*
	86	* On lockdep we dont want the hand-coded irq-enable of
	87	* _raw_spin_lock_flags() code, because lockdep assumes
	88	* that interrupts are not re-enabled during lock-acquire:
	89	*/
	90	#ifdef CONFIG_LOCKDEP
	91	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	92	#else
	93	_raw_spin_lock_flags(lock, &flags);
	94	#endif
	95	return flags;
	96	}
	97	EXPORT_SYMBOL(_spin_lock_irqsave);
	98
	99	void __lockfunc _spin_lock_irq(spinlock_t *lock)
	100	{
	101	local_irq_disable();
	102	preempt_disable();
	103	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	104	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	105	}
	106	EXPORT_SYMBOL(_spin_lock_irq);
	107
	108	void __lockfunc _spin_lock_bh(spinlock_t *lock)
	109	{
	110	local_bh_disable();
	111	preempt_disable();
	112	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	113	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	114	}
	115	EXPORT_SYMBOL(_spin_lock_bh);
	116
	117	unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
	118	{
	119	unsigned long flags;
	120
	121	local_irq_save(flags);
	122	preempt_disable();
	123	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	124	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
	125	return flags;
	126	}
	127	EXPORT_SYMBOL(_read_lock_irqsave);
	128
	129	void __lockfunc _read_lock_irq(rwlock_t *lock)
	130	{
	131	local_irq_disable();
	132	preempt_disable();
	133	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	134	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
	135	}
	136	EXPORT_SYMBOL(_read_lock_irq);
	137
	138	void __lockfunc _read_lock_bh(rwlock_t *lock)
	139	{
	140	local_bh_disable();
	141	preempt_disable();
	142	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	143	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
	144	}
	145	EXPORT_SYMBOL(_read_lock_bh);
	146
	147	unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
	148	{
	149	unsigned long flags;
	150
	151	local_irq_save(flags);
	152	preempt_disable();
	153	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	154	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
	155	return flags;
	156	}
	157	EXPORT_SYMBOL(_write_lock_irqsave);
	158
	159	void __lockfunc _write_lock_irq(rwlock_t *lock)
	160	{
	161	local_irq_disable();
	162	preempt_disable();
	163	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	164	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
	165	}
	166	EXPORT_SYMBOL(_write_lock_irq);
	167
	168	void __lockfunc _write_lock_bh(rwlock_t *lock)
	169	{
	170	local_bh_disable();
	171	preempt_disable();
	172	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	173	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
	174	}
	175	EXPORT_SYMBOL(_write_lock_bh);
	176
	177	void __lockfunc _spin_lock(spinlock_t *lock)
	178	{
	179	preempt_disable();
	180	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	181	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	182	}
	183
	184	EXPORT_SYMBOL(_spin_lock);
	185
	186	void __lockfunc _write_lock(rwlock_t *lock)
	187	{
	188	preempt_disable();
	189	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	190	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
	191	}
	192
	193	EXPORT_SYMBOL(_write_lock);
	194
	195	#else /* CONFIG_PREEMPT: */
	196
	197	/*
	198	* This could be a long-held lock. We both prepare to spin for a long
	199	* time (making _this_ CPU preemptable if possible), and we also signal
	200	* towards that other CPU that it should break the lock ASAP.
	201	*
	202	* (We do this in a function because inlining it would be excessive.)
	203	*/
	204
	205	#define BUILD_LOCK_OPS(op, locktype) \
	206	void __lockfunc _##op##_lock(locktype##_t *lock) \
	207	{ \
	208	for (;;) { \
	209	preempt_disable(); \
	210	if (likely(_raw_##op##_trylock(lock))) \
	211	break; \
	212	preempt_enable(); \
	213	\
	214	if (!(lock)->break_lock) \
	215	(lock)->break_lock = 1; \
	216	while (!op##_can_lock(lock) && (lock)->break_lock) \
	217	_raw_##op##_relax(&lock->raw_lock); \
	218	} \
	219	(lock)->break_lock = 0; \
	220	} \
	221	\
	222	EXPORT_SYMBOL(_##op##_lock); \
	223	\
	224	unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
	225	{ \
	226	unsigned long flags; \
	227	\
	228	for (;;) { \
	229	preempt_disable(); \
	230	local_irq_save(flags); \
	231	if (likely(_raw_##op##_trylock(lock))) \
	232	break; \
	233	local_irq_restore(flags); \
	234	preempt_enable(); \
	235	\
	236	if (!(lock)->break_lock) \
	237	(lock)->break_lock = 1; \
	238	while (!op##_can_lock(lock) && (lock)->break_lock) \
	239	_raw_##op##_relax(&lock->raw_lock); \
	240	} \
	241	(lock)->break_lock = 0; \
	242	return flags; \
	243	} \
	244	\
	245	EXPORT_SYMBOL(_##op##_lock_irqsave); \
	246	\
	247	void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
	248	{ \
	249	_##op##_lock_irqsave(lock); \
	250	} \
	251	\
	252	EXPORT_SYMBOL(_##op##_lock_irq); \
	253	\
	254	void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
	255	{ \
	256	unsigned long flags; \
	257	\
	258	/* */ \
	259	/* Careful: we must exclude softirqs too, hence the */ \
	260	/* irq-disabling. We use the generic preemption-aware */ \
	261	/* function: */ \
	262	/**/ \
	263	flags = _##op##_lock_irqsave(lock); \
	264	local_bh_disable(); \
	265	local_irq_restore(flags); \
	266	} \
	267	\
	268	EXPORT_SYMBOL(_##op##_lock_bh)
	269
	270	/*
	271	* Build preemption-friendly versions of the following
	272	* lock-spinning functions:
	273	*
	274	* _[spin\|read\|write]_lock()
	275	* _[spin\|read\|write]_lock_irq()
	276	* _[spin\|read\|write]_lock_irqsave()
	277	* _[spin\|read\|write]_lock_bh()
	278	*/
	279	BUILD_LOCK_OPS(spin, spinlock);
	280	BUILD_LOCK_OPS(read, rwlock);
	281	BUILD_LOCK_OPS(write, rwlock);
	282
	283	#endif /* CONFIG_PREEMPT */
	284
	285	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	286
	287	void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
	288	{
	289	preempt_disable();
	290	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	291	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	292	}
	293	EXPORT_SYMBOL(_spin_lock_nested);
	294
	295	unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
	296	{
	297	unsigned long flags;
	298
	299	local_irq_save(flags);
	300	preempt_disable();
	301	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	302	/*
	303	* On lockdep we dont want the hand-coded irq-enable of
	304	* _raw_spin_lock_flags() code, because lockdep assumes
	305	* that interrupts are not re-enabled during lock-acquire:
	306	*/
	307	#ifdef CONFIG_LOCKDEP
	308	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	309	#else
	310	_raw_spin_lock_flags(lock, &flags);
	311	#endif
	312	return flags;
	313	}
	314	EXPORT_SYMBOL(_spin_lock_irqsave_nested);
	315
	316	void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
	317	struct lockdep_map *nest_lock)
	318	{
	319	preempt_disable();
	320	spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
	321	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	322	}
	323	EXPORT_SYMBOL(_spin_lock_nest_lock);
	324
	325	#endif
	326
	327	void __lockfunc _spin_unlock(spinlock_t *lock)
	328	{
	329	spin_release(&lock->dep_map, 1, _RET_IP_);
	330	_raw_spin_unlock(lock);
	331	preempt_enable();
	332	}
	333	EXPORT_SYMBOL(_spin_unlock);
	334
	335	void __lockfunc _write_unlock(rwlock_t *lock)
	336	{
	337	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	338	_raw_write_unlock(lock);
	339	preempt_enable();
	340	}
	341	EXPORT_SYMBOL(_write_unlock);
	342
	343	void __lockfunc _read_unlock(rwlock_t *lock)
	344	{
	345	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	346	_raw_read_unlock(lock);
	347	preempt_enable();
	348	}
	349	EXPORT_SYMBOL(_read_unlock);
	350
	351	void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
	352	{
	353	spin_release(&lock->dep_map, 1, _RET_IP_);
	354	_raw_spin_unlock(lock);
	355	local_irq_restore(flags);
	356	preempt_enable();
	357	}
	358	EXPORT_SYMBOL(_spin_unlock_irqrestore);
	359
	360	void __lockfunc _spin_unlock_irq(spinlock_t *lock)
	361	{
	362	spin_release(&lock->dep_map, 1, _RET_IP_);
	363	_raw_spin_unlock(lock);
	364	local_irq_enable();
	365	preempt_enable();
	366	}
	367	EXPORT_SYMBOL(_spin_unlock_irq);
	368
	369	void __lockfunc _spin_unlock_bh(spinlock_t *lock)
	370	{
	371	spin_release(&lock->dep_map, 1, _RET_IP_);
	372	_raw_spin_unlock(lock);
	373	preempt_enable_no_resched();
	374	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	375	}
	376	EXPORT_SYMBOL(_spin_unlock_bh);
	377
	378	void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	379	{
	380	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	381	_raw_read_unlock(lock);
	382	local_irq_restore(flags);
	383	preempt_enable();
	384	}
	385	EXPORT_SYMBOL(_read_unlock_irqrestore);
	386
	387	void __lockfunc _read_unlock_irq(rwlock_t *lock)
	388	{
	389	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	390	_raw_read_unlock(lock);
	391	local_irq_enable();
	392	preempt_enable();
	393	}
	394	EXPORT_SYMBOL(_read_unlock_irq);
	395
	396	void __lockfunc _read_unlock_bh(rwlock_t *lock)
	397	{
	398	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	399	_raw_read_unlock(lock);
	400	preempt_enable_no_resched();
	401	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	402	}
	403	EXPORT_SYMBOL(_read_unlock_bh);
	404
	405	void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	406	{
	407	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	408	_raw_write_unlock(lock);
	409	local_irq_restore(flags);
	410	preempt_enable();
	411	}
	412	EXPORT_SYMBOL(_write_unlock_irqrestore);
	413
	414	void __lockfunc _write_unlock_irq(rwlock_t *lock)
	415	{
	416	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	417	_raw_write_unlock(lock);
	418	local_irq_enable();
	419	preempt_enable();
	420	}
	421	EXPORT_SYMBOL(_write_unlock_irq);
	422
	423	void __lockfunc _write_unlock_bh(rwlock_t *lock)
	424	{
	425	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	426	_raw_write_unlock(lock);
	427	preempt_enable_no_resched();
	428	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	429	}
	430	EXPORT_SYMBOL(_write_unlock_bh);
	431
	432	int __lockfunc _spin_trylock_bh(spinlock_t *lock)
	433	{
	434	local_bh_disable();
	435	preempt_disable();
	436	if (_raw_spin_trylock(lock)) {
	437	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
	438	return 1;
	439	}
	440
	441	preempt_enable_no_resched();
	442	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	443	return 0;
	444	}
	445	EXPORT_SYMBOL(_spin_trylock_bh);
	446
	447	notrace int in_lock_functions(unsigned long addr)
	448	{
	449	/* Linker adds these: start and end of __lockfunc functions */
	450	extern char __lock_text_start[], __lock_text_end[];
	451
	452	return addr >= (unsigned long)__lock_text_start
	453	&& addr < (unsigned long)__lock_text_end;
	454	}
	455	EXPORT_SYMBOL(in_lock_functions);