[deliverable/linux.git] / include / asm-generic / qspinlock.h

/*
 * Queued spinlock
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
 * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
 *
 * Authors: Waiman Long <waiman.long@hpe.com>
 */
#ifndef __ASM_GENERIC_QSPINLOCK_H
#define __ASM_GENERIC_QSPINLOCK_H

#include <asm-generic/qspinlock_types.h>

/**
 * queued_spin_is_locked - is the spinlock locked?
 * @lock: Pointer to queued spinlock structure
 * Return: 1 if it is locked, 0 otherwise
 */
static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
{
	/*
	 * queued_spin_lock_slowpath() can ACQUIRE the lock before
	 * issuing the unordered store that sets _Q_LOCKED_VAL.
	 *
	 * See both smp_cond_acquire() sites for more detail.
	 *
	 * This however means that in code like:
	 *
	 *   spin_lock(A)		spin_lock(B)
	 *   spin_unlock_wait(B)	spin_is_locked(A)
	 *   do_something()		do_something()
	 *
	 * Both CPUs can end up running do_something() because the store
	 * setting _Q_LOCKED_VAL will pass through the loads in
	 * spin_unlock_wait() and/or spin_is_locked().
	 *
	 * Avoid this by issuing a full memory barrier between the spin_lock()
	 * and the loads in spin_unlock_wait() and spin_is_locked().
	 *
	 * Note that regular mutual exclusion doesn't care about this
	 * delayed store.
	 */
	smp_mb();
	return atomic_read(&lock->val) & _Q_LOCKED_MASK;
}

/**
 * queued_spin_value_unlocked - is the spinlock structure unlocked?
 * @lock: queued spinlock structure
 * Return: 1 if it is unlocked, 0 otherwise
 *
 * N.B. Whenever there are tasks waiting for the lock, it is considered
 *      locked wrt the lockref code to avoid lock stealing by the lockref
 *      code and change things underneath the lock. This also allows some
 *      optimizations to be applied without conflict with lockref.
 */
static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
{
	return !atomic_read(&lock.val);
}

/**
 * queued_spin_is_contended - check if the lock is contended
 * @lock : Pointer to queued spinlock structure
 * Return: 1 if lock contended, 0 otherwise
 */
static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
{
	return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
}
/**
 * queued_spin_trylock - try to acquire the queued spinlock
 * @lock : Pointer to queued spinlock structure
 * Return: 1 if lock acquired, 0 if failed
 */
static __always_inline int queued_spin_trylock(struct qspinlock *lock)
{
	if (!atomic_read(&lock->val) &&
	   (atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL) == 0))
		return 1;
	return 0;
}

extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);

/**
 * queued_spin_lock - acquire a queued spinlock
 * @lock: Pointer to queued spinlock structure
 */
static __always_inline void queued_spin_lock(struct qspinlock *lock)
{
	u32 val;

	val = atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL);
	if (likely(val == 0))
		return;
	queued_spin_lock_slowpath(lock, val);
}

#ifndef queued_spin_unlock
/**
 * queued_spin_unlock - release a queued spinlock
 * @lock : Pointer to queued spinlock structure
 */
static __always_inline void queued_spin_unlock(struct qspinlock *lock)
{
	/*
	 * smp_mb__before_atomic() in order to guarantee release semantics
	 */
	smp_mb__before_atomic();
	atomic_sub(_Q_LOCKED_VAL, &lock->val);
}
#endif

/**
 * queued_spin_unlock_wait - wait until current lock holder releases the lock
 * @lock : Pointer to queued spinlock structure
 *
 * There is a very slight possibility of live-lock if the lockers keep coming
 * and the waiter is just unfortunate enough to not see any unlock state.
 */
static inline void queued_spin_unlock_wait(struct qspinlock *lock)
{
	/* See queued_spin_is_locked() */
	smp_mb();
	while (atomic_read(&lock->val) & _Q_LOCKED_MASK)
		cpu_relax();
}

#ifndef virt_spin_lock
static __always_inline bool virt_spin_lock(struct qspinlock *lock)
{
	return false;
}
#endif

/*
 * Remapping spinlock architecture specific functions to the corresponding
 * queued spinlock functions.
 */
#define arch_spin_is_locked(l)		queued_spin_is_locked(l)
#define arch_spin_is_contended(l)	queued_spin_is_contended(l)
#define arch_spin_value_unlocked(l)	queued_spin_value_unlocked(l)
#define arch_spin_lock(l)		queued_spin_lock(l)
#define arch_spin_trylock(l)		queued_spin_trylock(l)
#define arch_spin_unlock(l)		queued_spin_unlock(l)
#define arch_spin_lock_flags(l, f)	queued_spin_lock(l)
#define arch_spin_unlock_wait(l)	queued_spin_unlock_wait(l)

#endif /* __ASM_GENERIC_QSPINLOCK_H */
Commit	Line	Data
a33fda35 WL	1	/*
	2	* Queued spinlock
	3	*
	4	* This program is free software; you can redistribute it and/or modify
	5	* it under the terms of the GNU General Public License as published by
	6	* the Free Software Foundation; either version 2 of the License, or
	7	* (at your option) any later version.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
64d816cb	15	* (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
a33fda35	16	*
64d816cb	17	* Authors: Waiman Long <waiman.long@hpe.com>
a33fda35 WL	18	*/
	19	#ifndef __ASM_GENERIC_QSPINLOCK_H
	20	#define __ASM_GENERIC_QSPINLOCK_H
	21
	22	#include <asm-generic/qspinlock_types.h>
	23
	24	/**
	25	* queued_spin_is_locked - is the spinlock locked?
	26	* @lock: Pointer to queued spinlock structure
	27	* Return: 1 if it is locked, 0 otherwise
	28	*/
	29	static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
	30	{
54cf809b PZ	31	/*
	32	* queued_spin_lock_slowpath() can ACQUIRE the lock before
	33	* issuing the unordered store that sets _Q_LOCKED_VAL.
	34	*
	35	* See both smp_cond_acquire() sites for more detail.
	36	*
	37	* This however means that in code like:
	38	*
	39	* spin_lock(A) spin_lock(B)
	40	* spin_unlock_wait(B) spin_is_locked(A)
	41	* do_something() do_something()
	42	*
	43	* Both CPUs can end up running do_something() because the store
	44	* setting _Q_LOCKED_VAL will pass through the loads in
	45	* spin_unlock_wait() and/or spin_is_locked().
	46	*
	47	* Avoid this by issuing a full memory barrier between the spin_lock()
	48	* and the loads in spin_unlock_wait() and spin_is_locked().
	49	*
	50	* Note that regular mutual exclusion doesn't care about this
	51	* delayed store.
	52	*/
	53	smp_mb();
	54	return atomic_read(&lock->val) & _Q_LOCKED_MASK;
a33fda35 WL	55	}
	56
	57	/**
	58	* queued_spin_value_unlocked - is the spinlock structure unlocked?
	59	* @lock: queued spinlock structure
	60	* Return: 1 if it is unlocked, 0 otherwise
	61	*
	62	* N.B. Whenever there are tasks waiting for the lock, it is considered
	63	* locked wrt the lockref code to avoid lock stealing by the lockref
	64	* code and change things underneath the lock. This also allows some
	65	* optimizations to be applied without conflict with lockref.
	66	*/
	67	static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
	68	{
	69	return !atomic_read(&lock.val);
	70	}
	71
	72	/**
	73	* queued_spin_is_contended - check if the lock is contended
	74	* @lock : Pointer to queued spinlock structure
	75	* Return: 1 if lock contended, 0 otherwise
	76	*/
	77	static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
	78	{
	79	return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
	80	}
	81	/**
	82	* queued_spin_trylock - try to acquire the queued spinlock
	83	* @lock : Pointer to queued spinlock structure
	84	* Return: 1 if lock acquired, 0 if failed
	85	*/
	86	static __always_inline int queued_spin_trylock(struct qspinlock *lock)
	87	{
	88	if (!atomic_read(&lock->val) &&
64d816cb	89	(atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL) == 0))
a33fda35 WL	90	return 1;
	91	return 0;
	92	}
	93
	94	extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
	95
	96	/**
	97	* queued_spin_lock - acquire a queued spinlock
	98	* @lock: Pointer to queued spinlock structure
	99	*/
	100	static __always_inline void queued_spin_lock(struct qspinlock *lock)
	101	{
	102	u32 val;
	103
64d816cb	104	val = atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL);
a33fda35 WL	105	if (likely(val == 0))
	106	return;
	107	queued_spin_lock_slowpath(lock, val);
	108	}
	109
	110	#ifndef queued_spin_unlock
	111	/**
	112	* queued_spin_unlock - release a queued spinlock
	113	* @lock : Pointer to queued spinlock structure
	114	*/
	115	static __always_inline void queued_spin_unlock(struct qspinlock *lock)
	116	{
	117	/*
	118	* smp_mb__before_atomic() in order to guarantee release semantics
	119	*/
64d816cb	120	smp_mb__before_atomic();
a33fda35 WL	121	atomic_sub(_Q_LOCKED_VAL, &lock->val);
	122	}
	123	#endif
	124
	125	/**
	126	* queued_spin_unlock_wait - wait until current lock holder releases the lock
	127	* @lock : Pointer to queued spinlock structure
	128	*
	129	* There is a very slight possibility of live-lock if the lockers keep coming
	130	* and the waiter is just unfortunate enough to not see any unlock state.
	131	*/
	132	static inline void queued_spin_unlock_wait(struct qspinlock *lock)
	133	{
54cf809b PZ	134	/* See queued_spin_is_locked() */
54cf809b PZ	135	smp_mb();
a33fda35 WL	136	while (atomic_read(&lock->val) & _Q_LOCKED_MASK)
	137	cpu_relax();
	138	}
	139
43b3f028 PZ	140	#ifndef virt_spin_lock
43b3f028 PZ	141	static __always_inline bool virt_spin_lock(struct qspinlock *lock)
2aa79af6 PZI	142	{
	143	return false;
	144	}
	145	#endif
	146
a33fda35 WL	147	/*
	148	* Remapping spinlock architecture specific functions to the corresponding
	149	* queued spinlock functions.
	150	*/
	151	#define arch_spin_is_locked(l) queued_spin_is_locked(l)
	152	#define arch_spin_is_contended(l) queued_spin_is_contended(l)
	153	#define arch_spin_value_unlocked(l) queued_spin_value_unlocked(l)
	154	#define arch_spin_lock(l) queued_spin_lock(l)
	155	#define arch_spin_trylock(l) queued_spin_trylock(l)
	156	#define arch_spin_unlock(l) queued_spin_unlock(l)
	157	#define arch_spin_lock_flags(l, f) queued_spin_lock(l)
	158	#define arch_spin_unlock_wait(l) queued_spin_unlock_wait(l)
	159
	160	#endif /* __ASM_GENERIC_QSPINLOCK_H */