[deliverable/linux.git] / arch / s390 / lib / spinlock.c

/*
 *    Out of line spinlock code.
 *
 *    Copyright IBM Corp. 2004, 2006
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <asm/io.h>

int spin_retry = 1000;

/**
 * spin_retry= parameter
 */
static int __init spin_retry_setup(char *str)
{
	spin_retry = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("spin_retry=", spin_retry_setup);

void arch_spin_lock_wait(arch_spinlock_t *lp)
{
	unsigned int cpu = SPINLOCK_LOCKVAL;
	unsigned int owner;
	int count;

	while (1) {
		owner = ACCESS_ONCE(lp->lock);
		/* Try to get the lock if it is free. */
		if (!owner) {
			if (_raw_compare_and_swap(&lp->lock, 0, cpu))
				return;
			continue;
		}
		/* Check if the lock owner is running. */
		if (!smp_vcpu_scheduled(~owner)) {
			smp_yield_cpu(~owner);
			continue;
		}
		/* Loop for a while on the lock value. */
		count = spin_retry;
		do {
			owner = ACCESS_ONCE(lp->lock);
		} while (owner && count-- > 0);
		if (!owner)
			continue;
		/*
		 * For multiple layers of hypervisors, e.g. z/VM + LPAR
		 * yield the CPU if the lock is still unavailable.
		 */
		if (!MACHINE_IS_LPAR)
			smp_yield_cpu(~owner);
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait);

void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
{
	unsigned int cpu = SPINLOCK_LOCKVAL;
	unsigned int owner;
	int count;

	local_irq_restore(flags);
	while (1) {
		owner = ACCESS_ONCE(lp->lock);
		/* Try to get the lock if it is free. */
		if (!owner) {
			local_irq_disable();
			if (_raw_compare_and_swap(&lp->lock, 0, cpu))
				return;
			local_irq_restore(flags);
		}
		/* Check if the lock owner is running. */
		if (!smp_vcpu_scheduled(~owner)) {
			smp_yield_cpu(~owner);
			continue;
		}
		/* Loop for a while on the lock value. */
		count = spin_retry;
		do {
			owner = ACCESS_ONCE(lp->lock);
		} while (owner && count-- > 0);
		if (!owner)
			continue;
		/*
		 * For multiple layers of hypervisors, e.g. z/VM + LPAR
		 * yield the CPU if the lock is still unavailable.
		 */
		if (!MACHINE_IS_LPAR)
			smp_yield_cpu(~owner);
	}
}
EXPORT_SYMBOL(arch_spin_lock_wait_flags);

void arch_spin_relax(arch_spinlock_t *lp)
{
	unsigned int cpu = lp->lock;
	if (cpu != 0) {
		if (MACHINE_IS_VM || MACHINE_IS_KVM ||
		    !smp_vcpu_scheduled(~cpu))
			smp_yield_cpu(~cpu);
	}
}
EXPORT_SYMBOL(arch_spin_relax);

int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
	int count;

	for (count = spin_retry; count > 0; count--)
		if (arch_spin_trylock_once(lp))
			return 1;
	return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);

void _raw_read_lock_wait(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		old = ACCESS_ONCE(rw->lock);
		if ((int) old < 0)
			continue;
		if (_raw_compare_and_swap(&rw->lock, old, old + 1))
			return;
	}
}
EXPORT_SYMBOL(_raw_read_lock_wait);

void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
{
	unsigned int old;
	int count = spin_retry;

	local_irq_restore(flags);
	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		old = ACCESS_ONCE(rw->lock);
		if ((int) old < 0)
			continue;
		local_irq_disable();
		if (_raw_compare_and_swap(&rw->lock, old, old + 1))
			return;
		local_irq_restore(flags);
	}
}
EXPORT_SYMBOL(_raw_read_lock_wait_flags);

int _raw_read_trylock_retry(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (count-- > 0) {
		old = ACCESS_ONCE(rw->lock);
		if ((int) old < 0)
			continue;
		if (_raw_compare_and_swap(&rw->lock, old, old + 1))
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_read_trylock_retry);

void _raw_write_lock_wait(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		old = ACCESS_ONCE(rw->lock);
		if (old)
			continue;
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
			return;
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait);

void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
{
	unsigned int old;
	int count = spin_retry;

	local_irq_restore(flags);
	while (1) {
		if (count-- <= 0) {
			smp_yield();
			count = spin_retry;
		}
		old = ACCESS_ONCE(rw->lock);
		if (old)
			continue;
		local_irq_disable();
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
			return;
		local_irq_restore(flags);
	}
}
EXPORT_SYMBOL(_raw_write_lock_wait_flags);

int _raw_write_trylock_retry(arch_rwlock_t *rw)
{
	unsigned int old;
	int count = spin_retry;

	while (count-- > 0) {
		old = ACCESS_ONCE(rw->lock);
		if (old)
			continue;
		if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL(_raw_write_trylock_retry);
Commit	Line	Data
951f22d5	1	/*
951f22d5 MS	2	* Out of line spinlock code.
951f22d5 MS	3	*
a53c8fab	4	* Copyright IBM Corp. 2004, 2006
951f22d5 MS	5	* Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
	6	*/
	7
	8	#include <linux/types.h>
	9	#include <linux/module.h>
	10	#include <linux/spinlock.h>
	11	#include <linux/init.h>
8b646bd7	12	#include <linux/smp.h>
951f22d5 MS	13	#include <asm/io.h>
951f22d5 MS	14
951f22d5 MS	15	int spin_retry = 1000;
	16
	17	/**
	18	* spin_retry= parameter
	19	*/
	20	static int __init spin_retry_setup(char *str)
	21	{
	22	spin_retry = simple_strtoul(str, &str, 0);
	23	return 1;
	24	}
	25	__setup("spin_retry=", spin_retry_setup);
	26
0199c4e6	27	void arch_spin_lock_wait(arch_spinlock_t *lp)
951f22d5	28	{
6c8cd5bb	29	unsigned int cpu = SPINLOCK_LOCKVAL;
59b69787	30	unsigned int owner;
2e4006b3	31	int count;
951f22d5 MS	32
951f22d5 MS	33	while (1) {
470ada6b MS	34	owner = ACCESS_ONCE(lp->lock);
	35	/* Try to get the lock if it is free. */
	36	if (!owner) {
	37	if (_raw_compare_and_swap(&lp->lock, 0, cpu))
	38	return;
	39	continue;
951f22d5	40	}
470ada6b MS	41	/* Check if the lock owner is running. */
	42	if (!smp_vcpu_scheduled(~owner)) {
	43	smp_yield_cpu(~owner);
	44	continue;
	45	}
	46	/* Loop for a while on the lock value. */
	47	count = spin_retry;
	48	do {
	49	owner = ACCESS_ONCE(lp->lock);
	50	} while (owner && count-- > 0);
	51	if (!owner)
	52	continue;
	53	/*
	54	* For multiple layers of hypervisors, e.g. z/VM + LPAR
	55	* yield the CPU if the lock is still unavailable.
	56	*/
	57	if (!MACHINE_IS_LPAR)
8b646bd7	58	smp_yield_cpu(~owner);
951f22d5 MS	59	}
951f22d5 MS	60	}
0199c4e6	61	EXPORT_SYMBOL(arch_spin_lock_wait);
951f22d5	62
0199c4e6	63	void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
894cdde2	64	{
6c8cd5bb	65	unsigned int cpu = SPINLOCK_LOCKVAL;
59b69787	66	unsigned int owner;
2e4006b3	67	int count;
894cdde2 HH	68
	69	local_irq_restore(flags);
	70	while (1) {
470ada6b MS	71	owner = ACCESS_ONCE(lp->lock);
	72	/* Try to get the lock if it is free. */
	73	if (!owner) {
	74	local_irq_disable();
	75	if (_raw_compare_and_swap(&lp->lock, 0, cpu))
	76	return;
	77	local_irq_restore(flags);
	78	}
	79	/* Check if the lock owner is running. */
	80	if (!smp_vcpu_scheduled(~owner)) {
	81	smp_yield_cpu(~owner);
	82	continue;
894cdde2	83	}
470ada6b MS	84	/* Loop for a while on the lock value. */
	85	count = spin_retry;
	86	do {
	87	owner = ACCESS_ONCE(lp->lock);
	88	} while (owner && count-- > 0);
	89	if (!owner)
	90	continue;
	91	/*
	92	* For multiple layers of hypervisors, e.g. z/VM + LPAR
	93	* yield the CPU if the lock is still unavailable.
	94	*/
	95	if (!MACHINE_IS_LPAR)
8b646bd7	96	smp_yield_cpu(~owner);
894cdde2 HH	97	}
894cdde2 HH	98	}
0199c4e6	99	EXPORT_SYMBOL(arch_spin_lock_wait_flags);
894cdde2	100
5b3f683e PH	101	void arch_spin_relax(arch_spinlock_t *lp)
	102	{
	103	unsigned int cpu = lp->lock;
	104	if (cpu != 0) {
	105	if (MACHINE_IS_VM \|\| MACHINE_IS_KVM \|\|
	106	!smp_vcpu_scheduled(~cpu))
	107	smp_yield_cpu(~cpu);
	108	}
	109	}
	110	EXPORT_SYMBOL(arch_spin_relax);
	111
0199c4e6	112	int arch_spin_trylock_retry(arch_spinlock_t *lp)
951f22d5	113	{
3c1fcfe2	114	int count;
951f22d5	115
bae8f567	116	for (count = spin_retry; count > 0; count--)
5b3f683e	117	if (arch_spin_trylock_once(lp))
951f22d5	118	return 1;
951f22d5 MS	119	return 0;
951f22d5 MS	120	}
0199c4e6	121	EXPORT_SYMBOL(arch_spin_trylock_retry);
951f22d5	122
fb3a6bbc	123	void _raw_read_lock_wait(arch_rwlock_t *rw)
951f22d5 MS	124	{
	125	unsigned int old;
	126	int count = spin_retry;
	127
	128	while (1) {
	129	if (count-- <= 0) {
8b646bd7	130	smp_yield();
951f22d5 MS	131	count = spin_retry;
951f22d5 MS	132	}
bae8f567 MS	133	old = ACCESS_ONCE(rw->lock);
bae8f567 MS	134	if ((int) old < 0)
96567161	135	continue;
5b3f683e	136	if (_raw_compare_and_swap(&rw->lock, old, old + 1))
951f22d5 MS	137	return;
	138	}
	139	}
	140	EXPORT_SYMBOL(_raw_read_lock_wait);
	141
fb3a6bbc	142	void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
ce58ae6f HC	143	{
	144	unsigned int old;
	145	int count = spin_retry;
	146
	147	local_irq_restore(flags);
	148	while (1) {
	149	if (count-- <= 0) {
8b646bd7	150	smp_yield();
ce58ae6f HC	151	count = spin_retry;
ce58ae6f HC	152	}
bae8f567 MS	153	old = ACCESS_ONCE(rw->lock);
bae8f567 MS	154	if ((int) old < 0)
ce58ae6f	155	continue;
ce58ae6f	156	local_irq_disable();
5b3f683e	157	if (_raw_compare_and_swap(&rw->lock, old, old + 1))
ce58ae6f	158	return;
939c5ae4	159	local_irq_restore(flags);
ce58ae6f HC	160	}
	161	}
	162	EXPORT_SYMBOL(_raw_read_lock_wait_flags);
	163
fb3a6bbc	164	int _raw_read_trylock_retry(arch_rwlock_t *rw)
951f22d5 MS	165	{
	166	unsigned int old;
	167	int count = spin_retry;
	168
	169	while (count-- > 0) {
bae8f567 MS	170	old = ACCESS_ONCE(rw->lock);
bae8f567 MS	171	if ((int) old < 0)
96567161	172	continue;
5b3f683e	173	if (_raw_compare_and_swap(&rw->lock, old, old + 1))
951f22d5 MS	174	return 1;
	175	}
	176	return 0;
	177	}
	178	EXPORT_SYMBOL(_raw_read_trylock_retry);
	179
fb3a6bbc	180	void _raw_write_lock_wait(arch_rwlock_t *rw)
951f22d5	181	{
bae8f567	182	unsigned int old;
951f22d5 MS	183	int count = spin_retry;
	184
	185	while (1) {
	186	if (count-- <= 0) {
8b646bd7	187	smp_yield();
951f22d5 MS	188	count = spin_retry;
951f22d5 MS	189	}
bae8f567 MS	190	old = ACCESS_ONCE(rw->lock);
bae8f567 MS	191	if (old)
96567161	192	continue;
5b3f683e	193	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
951f22d5 MS	194	return;
	195	}
	196	}
	197	EXPORT_SYMBOL(_raw_write_lock_wait);
	198
fb3a6bbc	199	void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
ce58ae6f	200	{
bae8f567	201	unsigned int old;
ce58ae6f HC	202	int count = spin_retry;
	203
	204	local_irq_restore(flags);
	205	while (1) {
	206	if (count-- <= 0) {
8b646bd7	207	smp_yield();
ce58ae6f HC	208	count = spin_retry;
ce58ae6f HC	209	}
bae8f567 MS	210	old = ACCESS_ONCE(rw->lock);
bae8f567 MS	211	if (old)
ce58ae6f HC	212	continue;
ce58ae6f HC	213	local_irq_disable();
5b3f683e	214	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
ce58ae6f	215	return;
939c5ae4	216	local_irq_restore(flags);
ce58ae6f HC	217	}
	218	}
	219	EXPORT_SYMBOL(_raw_write_lock_wait_flags);
	220
fb3a6bbc	221	int _raw_write_trylock_retry(arch_rwlock_t *rw)
951f22d5	222	{
bae8f567	223	unsigned int old;
951f22d5 MS	224	int count = spin_retry;
	225
	226	while (count-- > 0) {
bae8f567 MS	227	old = ACCESS_ONCE(rw->lock);
bae8f567 MS	228	if (old)
96567161	229	continue;
5b3f683e	230	if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
951f22d5 MS	231	return 1;
	232	}
	233	return 0;
	234	}
	235	EXPORT_SYMBOL(_raw_write_trylock_retry);