blockdev: turn a rw semaphore into a percpu rw semaphore

[deliverable/linux.git] / include / linux / percpu-rwsem.h

#ifndef _LINUX_PERCPU_RWSEM_H
#define _LINUX_PERCPU_RWSEM_H

#include <linux/mutex.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/delay.h>

struct percpu_rw_semaphore {
	unsigned __percpu *counters;
	bool locked;
	struct mutex mtx;
};

static inline void percpu_down_read(struct percpu_rw_semaphore *p)
{
	rcu_read_lock();
	if (unlikely(p->locked)) {
		rcu_read_unlock();
		mutex_lock(&p->mtx);
		this_cpu_inc(*p->counters);
		mutex_unlock(&p->mtx);
		return;
	}
	this_cpu_inc(*p->counters);
	rcu_read_unlock();
}

static inline void percpu_up_read(struct percpu_rw_semaphore *p)
{
	/*
	 * On X86, write operation in this_cpu_dec serves as a memory unlock
	 * barrier (i.e. memory accesses may be moved before the write, but
	 * no memory accesses are moved past the write).
	 * On other architectures this may not be the case, so we need smp_mb()
	 * there.
	 */
#if defined(CONFIG_X86) && (!defined(CONFIG_X86_PPRO_FENCE) && !defined(CONFIG_X86_OOSTORE))
	barrier();
#else
	smp_mb();
#endif
	this_cpu_dec(*p->counters);
}

static inline unsigned __percpu_count(unsigned __percpu *counters)
{
	unsigned total = 0;
	int cpu;

	for_each_possible_cpu(cpu)
		total += ACCESS_ONCE(*per_cpu_ptr(counters, cpu));

	return total;
}

static inline void percpu_down_write(struct percpu_rw_semaphore *p)
{
	mutex_lock(&p->mtx);
	p->locked = true;
	synchronize_rcu();
	while (__percpu_count(p->counters))
		msleep(1);
	smp_rmb(); /* paired with smp_mb() in percpu_sem_up_read() */
}

static inline void percpu_up_write(struct percpu_rw_semaphore *p)
{
	p->locked = false;
	mutex_unlock(&p->mtx);
}

static inline int percpu_init_rwsem(struct percpu_rw_semaphore *p)
{
	p->counters = alloc_percpu(unsigned);
	if (unlikely(!p->counters))
		return -ENOMEM;
	p->locked = false;
	mutex_init(&p->mtx);
	return 0;
}

static inline void percpu_free_rwsem(struct percpu_rw_semaphore *p)
{
	free_percpu(p->counters);
	p->counters = NULL; /* catch use after free bugs */
}

#endif