[deliverable/linux.git] / lib / percpu_counter.c

/*
 * Fast batching percpu counters.
 */

#include <linux/percpu_counter.h>
#include <linux/notifier.h>
#include <linux/mutex.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/module.h>

#ifdef CONFIG_HOTPLUG_CPU
static LIST_HEAD(percpu_counters);
static DEFINE_MUTEX(percpu_counters_lock);
#endif

void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
{
	int cpu;

	spin_lock(&fbc->lock);
	for_each_possible_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		*pcount = 0;
	}
	fbc->count = amount;
	spin_unlock(&fbc->lock);
}
EXPORT_SYMBOL(percpu_counter_set);

void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
{
	s64 count;
	s32 *pcount;
	int cpu = get_cpu();

	pcount = per_cpu_ptr(fbc->counters, cpu);
	count = *pcount + amount;
	if (count >= batch || count <= -batch) {
		spin_lock(&fbc->lock);
		fbc->count += count;
		*pcount = 0;
		spin_unlock(&fbc->lock);
	} else {
		*pcount = count;
	}
	put_cpu();
}
EXPORT_SYMBOL(__percpu_counter_add);

/*
 * Add up all the per-cpu counts, return the result.  This is a more accurate
 * but much slower version of percpu_counter_read_positive()
 */
s64 __percpu_counter_sum(struct percpu_counter *fbc)
{
	s64 ret;
	int cpu;

	spin_lock(&fbc->lock);
	ret = fbc->count;
	for_each_online_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		ret += *pcount;
	}
	spin_unlock(&fbc->lock);
	return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);

int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
{
	spin_lock_init(&fbc->lock);
	fbc->count = amount;
	fbc->counters = alloc_percpu(s32);
	if (!fbc->counters)
		return -ENOMEM;
#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_add(&fbc->list, &percpu_counters);
	mutex_unlock(&percpu_counters_lock);
#endif
	return 0;
}
EXPORT_SYMBOL(percpu_counter_init);

void percpu_counter_destroy(struct percpu_counter *fbc)
{
	if (!fbc->counters)
		return;

	free_percpu(fbc->counters);
#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_del(&fbc->list);
	mutex_unlock(&percpu_counters_lock);
#endif
}
EXPORT_SYMBOL(percpu_counter_destroy);

#ifdef CONFIG_HOTPLUG_CPU
static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
					unsigned long action, void *hcpu)
{
	unsigned int cpu;
	struct percpu_counter *fbc;

	if (action != CPU_DEAD)
		return NOTIFY_OK;

	cpu = (unsigned long)hcpu;
	mutex_lock(&percpu_counters_lock);
	list_for_each_entry(fbc, &percpu_counters, list) {
		s32 *pcount;

		spin_lock(&fbc->lock);
		pcount = per_cpu_ptr(fbc->counters, cpu);
		fbc->count += *pcount;
		*pcount = 0;
		spin_unlock(&fbc->lock);
	}
	mutex_unlock(&percpu_counters_lock);
	return NOTIFY_OK;
}

static int __init percpu_counter_startup(void)
{
	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	return 0;
}
module_init(percpu_counter_startup);
#endif
Commit	Line	Data
3cbc5640 RT	1	/*
	2	* Fast batching percpu counters.
	3	*/
	4
	5	#include <linux/percpu_counter.h>
c67ad917 AM	6	#include <linux/notifier.h>
	7	#include <linux/mutex.h>
	8	#include <linux/init.h>
	9	#include <linux/cpu.h>
3cbc5640 RT	10	#include <linux/module.h>
3cbc5640 RT	11
c67ad917 AM	12	#ifdef CONFIG_HOTPLUG_CPU
	13	static LIST_HEAD(percpu_counters);
	14	static DEFINE_MUTEX(percpu_counters_lock);
	15	#endif
	16
3a587f47 PZ	17	void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
	18	{
	19	int cpu;
	20
	21	spin_lock(&fbc->lock);
	22	for_each_possible_cpu(cpu) {
	23	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
	24	*pcount = 0;
	25	}
	26	fbc->count = amount;
	27	spin_unlock(&fbc->lock);
	28	}
	29	EXPORT_SYMBOL(percpu_counter_set);
	30
20e89767	31	void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
3cbc5640	32	{
20e89767	33	s64 count;
0216bfcf	34	s32 *pcount;
3cbc5640 RT	35	int cpu = get_cpu();
	36
	37	pcount = per_cpu_ptr(fbc->counters, cpu);
	38	count = *pcount + amount;
252e0ba6	39	if (count >= batch \|\| count <= -batch) {
3cbc5640 RT	40	spin_lock(&fbc->lock);
	41	fbc->count += count;
	42	*pcount = 0;
	43	spin_unlock(&fbc->lock);
	44	} else {
	45	*pcount = count;
	46	}
	47	put_cpu();
	48	}
252e0ba6	49	EXPORT_SYMBOL(__percpu_counter_add);
3cbc5640 RT	50
	51	/*
	52	* Add up all the per-cpu counts, return the result. This is a more accurate
	53	* but much slower version of percpu_counter_read_positive()
	54	*/
bf1d89c8	55	s64 __percpu_counter_sum(struct percpu_counter *fbc)
3cbc5640	56	{
0216bfcf	57	s64 ret;
3cbc5640 RT	58	int cpu;
	59
	60	spin_lock(&fbc->lock);
	61	ret = fbc->count;
b4ef0296	62	for_each_online_cpu(cpu) {
0216bfcf	63	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
3cbc5640 RT	64	ret += *pcount;
	65	}
	66	spin_unlock(&fbc->lock);
bf1d89c8	67	return ret;
3cbc5640	68	}
bf1d89c8	69	EXPORT_SYMBOL(__percpu_counter_sum);
c67ad917	70
833f4077	71	int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
c67ad917 AM	72	{
	73	spin_lock_init(&fbc->lock);
	74	fbc->count = amount;
	75	fbc->counters = alloc_percpu(s32);
833f4077 PZ	76	if (!fbc->counters)
833f4077 PZ	77	return -ENOMEM;
c67ad917 AM	78	#ifdef CONFIG_HOTPLUG_CPU
	79	mutex_lock(&percpu_counters_lock);
	80	list_add(&fbc->list, &percpu_counters);
	81	mutex_unlock(&percpu_counters_lock);
	82	#endif
833f4077	83	return 0;
c67ad917 AM	84	}
	85	EXPORT_SYMBOL(percpu_counter_init);
	86
	87	void percpu_counter_destroy(struct percpu_counter *fbc)
	88	{
833f4077 PZ	89	if (!fbc->counters)
	90	return;
	91
c67ad917 AM	92	free_percpu(fbc->counters);
	93	#ifdef CONFIG_HOTPLUG_CPU
	94	mutex_lock(&percpu_counters_lock);
	95	list_del(&fbc->list);
	96	mutex_unlock(&percpu_counters_lock);
	97	#endif
	98	}
	99	EXPORT_SYMBOL(percpu_counter_destroy);
	100
	101	#ifdef CONFIG_HOTPLUG_CPU
	102	static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
	103	unsigned long action, void *hcpu)
	104	{
	105	unsigned int cpu;
	106	struct percpu_counter *fbc;
	107
	108	if (action != CPU_DEAD)
	109	return NOTIFY_OK;
	110
	111	cpu = (unsigned long)hcpu;
	112	mutex_lock(&percpu_counters_lock);
	113	list_for_each_entry(fbc, &percpu_counters, list) {
	114	s32 *pcount;
	115
	116	spin_lock(&fbc->lock);
	117	pcount = per_cpu_ptr(fbc->counters, cpu);
	118	fbc->count += *pcount;
	119	*pcount = 0;
	120	spin_unlock(&fbc->lock);
	121	}
	122	mutex_unlock(&percpu_counters_lock);
	123	return NOTIFY_OK;
	124	}
	125
	126	static int __init percpu_counter_startup(void)
	127	{
	128	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	129	return 0;
	130	}
	131	module_init(percpu_counter_startup);
	132	#endif