[deliverable/linux.git] / kernel / irq / irqdesc.c

/*
 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
 * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
 *
 * This file contains the interrupt descriptor management code
 *
 * Detailed information is available in Documentation/DocBook/genericirq
 *
 */
#include <linux/irq.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/radix-tree.h>
#include <linux/bitmap.h>

#include "internals.h"

/*
 * lockdep: we want to handle all irq_desc locks as a single lock-class:
 */
static struct lock_class_key irq_desc_lock_class;

#if defined(CONFIG_SMP)
static void __init init_irq_default_affinity(void)
{
	alloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
	cpumask_setall(irq_default_affinity);
}
#else
static void __init init_irq_default_affinity(void)
{
}
#endif

#ifdef CONFIG_SMP
static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
{
	if (!zalloc_cpumask_var_node(&desc->irq_data.affinity, gfp, node))
		return -ENOMEM;

#ifdef CONFIG_GENERIC_PENDING_IRQ
	if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
		free_cpumask_var(desc->irq_data.affinity);
		return -ENOMEM;
	}
#endif
	return 0;
}

static void desc_smp_init(struct irq_desc *desc, int node)
{
	desc->irq_data.node = node;
	cpumask_copy(desc->irq_data.affinity, irq_default_affinity);
#ifdef CONFIG_GENERIC_PENDING_IRQ
	cpumask_clear(desc->pending_mask);
#endif
}

static inline int desc_node(struct irq_desc *desc)
{
	return desc->irq_data.node;
}

#else
static inline int
alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; }
static inline void desc_smp_init(struct irq_desc *desc, int node) { }
static inline int desc_node(struct irq_desc *desc) { return 0; }
#endif

static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
		struct module *owner)
{
	int cpu;

	desc->irq_data.irq = irq;
	desc->irq_data.chip = &no_irq_chip;
	desc->irq_data.chip_data = NULL;
	desc->irq_data.handler_data = NULL;
	desc->irq_data.msi_desc = NULL;
	irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
	desc->handle_irq = handle_bad_irq;
	desc->depth = 1;
	desc->irq_count = 0;
	desc->irqs_unhandled = 0;
	desc->name = NULL;
	desc->owner = owner;
	for_each_possible_cpu(cpu)
		*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
	desc_smp_init(desc, node);
}

int nr_irqs = NR_IRQS;
EXPORT_SYMBOL_GPL(nr_irqs);

static DEFINE_MUTEX(sparse_irq_lock);
static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);

#ifdef CONFIG_SPARSE_IRQ

static RADIX_TREE(irq_desc_tree, GFP_KERNEL);

static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
{
	radix_tree_insert(&irq_desc_tree, irq, desc);
}

struct irq_desc *irq_to_desc(unsigned int irq)
{
	return radix_tree_lookup(&irq_desc_tree, irq);
}
EXPORT_SYMBOL(irq_to_desc);

static void delete_irq_desc(unsigned int irq)
{
	radix_tree_delete(&irq_desc_tree, irq);
}

#ifdef CONFIG_SMP
static void free_masks(struct irq_desc *desc)
{
#ifdef CONFIG_GENERIC_PENDING_IRQ
	free_cpumask_var(desc->pending_mask);
#endif
	free_cpumask_var(desc->irq_data.affinity);
}
#else
static inline void free_masks(struct irq_desc *desc) { }
#endif

static struct irq_desc *alloc_desc(int irq, int node, struct module *owner)
{
	struct irq_desc *desc;
	gfp_t gfp = GFP_KERNEL;

	desc = kzalloc_node(sizeof(*desc), gfp, node);
	if (!desc)
		return NULL;
	/* allocate based on nr_cpu_ids */
	desc->kstat_irqs = alloc_percpu(unsigned int);
	if (!desc->kstat_irqs)
		goto err_desc;

	if (alloc_masks(desc, gfp, node))
		goto err_kstat;

	raw_spin_lock_init(&desc->lock);
	lockdep_set_class(&desc->lock, &irq_desc_lock_class);

	desc_set_defaults(irq, desc, node, owner);

	return desc;

err_kstat:
	free_percpu(desc->kstat_irqs);
err_desc:
	kfree(desc);
	return NULL;
}

static void free_desc(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	unregister_irq_proc(irq, desc);

	mutex_lock(&sparse_irq_lock);
	delete_irq_desc(irq);
	mutex_unlock(&sparse_irq_lock);

	free_masks(desc);
	free_percpu(desc->kstat_irqs);
	kfree(desc);
}

static int alloc_descs(unsigned int start, unsigned int cnt, int node,
		       struct module *owner)
{
	struct irq_desc *desc;
	int i;

	for (i = 0; i < cnt; i++) {
		desc = alloc_desc(start + i, node, owner);
		if (!desc)
			goto err;
		mutex_lock(&sparse_irq_lock);
		irq_insert_desc(start + i, desc);
		mutex_unlock(&sparse_irq_lock);
	}
	return start;

err:
	for (i--; i >= 0; i--)
		free_desc(start + i);

	mutex_lock(&sparse_irq_lock);
	bitmap_clear(allocated_irqs, start, cnt);
	mutex_unlock(&sparse_irq_lock);
	return -ENOMEM;
}

static int irq_expand_nr_irqs(unsigned int nr)
{
	if (nr > IRQ_BITMAP_BITS)
		return -ENOMEM;
	nr_irqs = nr;
	return 0;
}

int __init early_irq_init(void)
{
	int i, initcnt, node = first_online_node;
	struct irq_desc *desc;

	init_irq_default_affinity();

	/* Let arch update nr_irqs and return the nr of preallocated irqs */
	initcnt = arch_probe_nr_irqs();
	printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt);

	if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
		nr_irqs = IRQ_BITMAP_BITS;

	if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
		initcnt = IRQ_BITMAP_BITS;

	if (initcnt > nr_irqs)
		nr_irqs = initcnt;

	for (i = 0; i < initcnt; i++) {
		desc = alloc_desc(i, node, NULL);
		set_bit(i, allocated_irqs);
		irq_insert_desc(i, desc);
	}
	return arch_early_irq_init();
}

#else /* !CONFIG_SPARSE_IRQ */

struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
	[0 ... NR_IRQS-1] = {
		.handle_irq	= handle_bad_irq,
		.depth		= 1,
		.lock		= __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
	}
};

int __init early_irq_init(void)
{
	int count, i, node = first_online_node;
	struct irq_desc *desc;

	init_irq_default_affinity();

	printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);

	desc = irq_desc;
	count = ARRAY_SIZE(irq_desc);

	for (i = 0; i < count; i++) {
		desc[i].kstat_irqs = alloc_percpu(unsigned int);
		alloc_masks(&desc[i], GFP_KERNEL, node);
		raw_spin_lock_init(&desc[i].lock);
		lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
		desc_set_defaults(i, &desc[i], node, NULL);
	}
	return arch_early_irq_init();
}

struct irq_desc *irq_to_desc(unsigned int irq)
{
	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
}
EXPORT_SYMBOL(irq_to_desc);

static void free_desc(unsigned int irq)
{
	dynamic_irq_cleanup(irq);
}

static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
			      struct module *owner)
{
	u32 i;

	for (i = 0; i < cnt; i++) {
		struct irq_desc *desc = irq_to_desc(start + i);

		desc->owner = owner;
	}
	return start;
}

static int irq_expand_nr_irqs(unsigned int nr)
{
	return -ENOMEM;
}

#endif /* !CONFIG_SPARSE_IRQ */

/**
 * generic_handle_irq - Invoke the handler for a particular irq
 * @irq:	The irq number to handle
 *
 */
int generic_handle_irq(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	if (!desc)
		return -EINVAL;
	generic_handle_irq_desc(irq, desc);
	return 0;
}
EXPORT_SYMBOL_GPL(generic_handle_irq);

/* Dynamic interrupt handling */

/**
 * irq_free_descs - free irq descriptors
 * @from:	Start of descriptor range
 * @cnt:	Number of consecutive irqs to free
 */
void irq_free_descs(unsigned int from, unsigned int cnt)
{
	int i;

	if (from >= nr_irqs || (from + cnt) > nr_irqs)
		return;

	for (i = 0; i < cnt; i++)
		free_desc(from + i);

	mutex_lock(&sparse_irq_lock);
	bitmap_clear(allocated_irqs, from, cnt);
	mutex_unlock(&sparse_irq_lock);
}
EXPORT_SYMBOL_GPL(irq_free_descs);

/**
 * irq_alloc_descs - allocate and initialize a range of irq descriptors
 * @irq:	Allocate for specific irq number if irq >= 0
 * @from:	Start the search from this irq number
 * @cnt:	Number of consecutive irqs to allocate.
 * @node:	Preferred node on which the irq descriptor should be allocated
 * @owner:	Owning module (can be NULL)
 *
 * Returns the first irq number or error code
 */
int __ref
__irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
		  struct module *owner)
{
	int start, ret;

	if (!cnt)
		return -EINVAL;

	if (irq >= 0) {
		if (from > irq)
			return -EINVAL;
		from = irq;
	}

	mutex_lock(&sparse_irq_lock);

	start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
					   from, cnt, 0);
	ret = -EEXIST;
	if (irq >=0 && start != irq)
		goto err;

	if (start + cnt > nr_irqs) {
		ret = irq_expand_nr_irqs(start + cnt);
		if (ret)
			goto err;
	}

	bitmap_set(allocated_irqs, start, cnt);
	mutex_unlock(&sparse_irq_lock);
	return alloc_descs(start, cnt, node, owner);

err:
	mutex_unlock(&sparse_irq_lock);
	return ret;
}
EXPORT_SYMBOL_GPL(__irq_alloc_descs);

/**
 * irq_reserve_irqs - mark irqs allocated
 * @from:	mark from irq number
 * @cnt:	number of irqs to mark
 *
 * Returns 0 on success or an appropriate error code
 */
int irq_reserve_irqs(unsigned int from, unsigned int cnt)
{
	unsigned int start;
	int ret = 0;

	if (!cnt || (from + cnt) > nr_irqs)
		return -EINVAL;

	mutex_lock(&sparse_irq_lock);
	start = bitmap_find_next_zero_area(allocated_irqs, nr_irqs, from, cnt, 0);
	if (start == from)
		bitmap_set(allocated_irqs, start, cnt);
	else
		ret = -EEXIST;
	mutex_unlock(&sparse_irq_lock);
	return ret;
}

/**
 * irq_get_next_irq - get next allocated irq number
 * @offset:	where to start the search
 *
 * Returns next irq number after offset or nr_irqs if none is found.
 */
unsigned int irq_get_next_irq(unsigned int offset)
{
	return find_next_bit(allocated_irqs, nr_irqs, offset);
}

struct irq_desc *
__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
		    unsigned int check)
{
	struct irq_desc *desc = irq_to_desc(irq);

	if (desc) {
		if (check & _IRQ_DESC_CHECK) {
			if ((check & _IRQ_DESC_PERCPU) &&
			    !irq_settings_is_per_cpu_devid(desc))
				return NULL;

			if (!(check & _IRQ_DESC_PERCPU) &&
			    irq_settings_is_per_cpu_devid(desc))
				return NULL;
		}

		if (bus)
			chip_bus_lock(desc);
		raw_spin_lock_irqsave(&desc->lock, *flags);
	}
	return desc;
}

void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
{
	raw_spin_unlock_irqrestore(&desc->lock, flags);
	if (bus)
		chip_bus_sync_unlock(desc);
}

int irq_set_percpu_devid(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	if (!desc)
		return -EINVAL;

	if (desc->percpu_enabled)
		return -EINVAL;

	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);

	if (!desc->percpu_enabled)
		return -ENOMEM;

	irq_set_percpu_devid_flags(irq);
	return 0;
}

/**
 * dynamic_irq_cleanup - cleanup a dynamically allocated irq
 * @irq:	irq number to initialize
 */
void dynamic_irq_cleanup(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);
	unsigned long flags;

	raw_spin_lock_irqsave(&desc->lock, flags);
	desc_set_defaults(irq, desc, desc_node(desc), NULL);
	raw_spin_unlock_irqrestore(&desc->lock, flags);
}

void kstat_incr_irq_this_cpu(unsigned int irq)
{
	kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
}

unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
{
	struct irq_desc *desc = irq_to_desc(irq);

	return desc && desc->kstat_irqs ?
			*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
}

unsigned int kstat_irqs(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);
	int cpu;
	int sum = 0;

	if (!desc || !desc->kstat_irqs)
		return 0;
	for_each_possible_cpu(cpu)
		sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
	return sum;
}
Commit	Line	Data
3795de23 TG	1	/*
	2	* Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
	3	* Copyright (C) 2005-2006, Thomas Gleixner, Russell King
	4	*
	5	* This file contains the interrupt descriptor management code
	6	*
	7	* Detailed information is available in Documentation/DocBook/genericirq
	8	*
	9	*/
	10	#include <linux/irq.h>
	11	#include <linux/slab.h>
ec53cf23	12	#include <linux/export.h>
3795de23 TG	13	#include <linux/interrupt.h>
	14	#include <linux/kernel_stat.h>
	15	#include <linux/radix-tree.h>
1f5a5b87	16	#include <linux/bitmap.h>
3795de23 TG	17
	18	#include "internals.h"
	19
	20	/*
	21	* lockdep: we want to handle all irq_desc locks as a single lock-class:
	22	*/
78f90d91	23	static struct lock_class_key irq_desc_lock_class;
3795de23	24
fe051434	25	#if defined(CONFIG_SMP)
3795de23 TG	26	static void __init init_irq_default_affinity(void)
	27	{
	28	alloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
	29	cpumask_setall(irq_default_affinity);
	30	}
	31	#else
	32	static void __init init_irq_default_affinity(void)
	33	{
	34	}
	35	#endif
	36
1f5a5b87 TG	37	#ifdef CONFIG_SMP
	38	static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
	39	{
	40	if (!zalloc_cpumask_var_node(&desc->irq_data.affinity, gfp, node))
	41	return -ENOMEM;
	42
	43	#ifdef CONFIG_GENERIC_PENDING_IRQ
	44	if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
	45	free_cpumask_var(desc->irq_data.affinity);
	46	return -ENOMEM;
	47	}
	48	#endif
	49	return 0;
	50	}
	51
	52	static void desc_smp_init(struct irq_desc *desc, int node)
	53	{
aa99ec0f	54	desc->irq_data.node = node;
1f5a5b87	55	cpumask_copy(desc->irq_data.affinity, irq_default_affinity);
b7b29338 TG	56	#ifdef CONFIG_GENERIC_PENDING_IRQ
	57	cpumask_clear(desc->pending_mask);
	58	#endif
	59	}
	60
	61	static inline int desc_node(struct irq_desc *desc)
	62	{
	63	return desc->irq_data.node;
1f5a5b87 TG	64	}
	65
	66	#else
	67	static inline int
	68	alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; }
	69	static inline void desc_smp_init(struct irq_desc *desc, int node) { }
b7b29338	70	static inline int desc_node(struct irq_desc *desc) { return 0; }
1f5a5b87 TG	71	#endif
1f5a5b87 TG	72
b6873807 SAS	73	static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
b6873807 SAS	74	struct module *owner)
1f5a5b87	75	{
6c9ae009 ED	76	int cpu;
6c9ae009 ED	77
1f5a5b87 TG	78	desc->irq_data.irq = irq;
	79	desc->irq_data.chip = &no_irq_chip;
	80	desc->irq_data.chip_data = NULL;
	81	desc->irq_data.handler_data = NULL;
	82	desc->irq_data.msi_desc = NULL;
f9e4989e	83	irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
801a0e9a	84	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
1f5a5b87 TG	85	desc->handle_irq = handle_bad_irq;
1f5a5b87 TG	86	desc->depth = 1;
b7b29338 TG	87	desc->irq_count = 0;
b7b29338 TG	88	desc->irqs_unhandled = 0;
1f5a5b87	89	desc->name = NULL;
b6873807	90	desc->owner = owner;
6c9ae009 ED	91	for_each_possible_cpu(cpu)
6c9ae009 ED	92	*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
1f5a5b87 TG	93	desc_smp_init(desc, node);
	94	}
	95
3795de23 TG	96	int nr_irqs = NR_IRQS;
	97	EXPORT_SYMBOL_GPL(nr_irqs);
	98
a05a900a	99	static DEFINE_MUTEX(sparse_irq_lock);
c1ee6264	100	static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);
1f5a5b87	101
3795de23 TG	102	#ifdef CONFIG_SPARSE_IRQ
3795de23 TG	103
baa0d233	104	static RADIX_TREE(irq_desc_tree, GFP_KERNEL);
3795de23	105
1f5a5b87	106	static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
3795de23 TG	107	{
	108	radix_tree_insert(&irq_desc_tree, irq, desc);
	109	}
	110
	111	struct irq_desc *irq_to_desc(unsigned int irq)
	112	{
	113	return radix_tree_lookup(&irq_desc_tree, irq);
	114	}
3911ff30	115	EXPORT_SYMBOL(irq_to_desc);
3795de23	116
1f5a5b87 TG	117	static void delete_irq_desc(unsigned int irq)
	118	{
	119	radix_tree_delete(&irq_desc_tree, irq);
	120	}
	121
	122	#ifdef CONFIG_SMP
	123	static void free_masks(struct irq_desc *desc)
	124	{
	125	#ifdef CONFIG_GENERIC_PENDING_IRQ
	126	free_cpumask_var(desc->pending_mask);
	127	#endif
c0a19ebc	128	free_cpumask_var(desc->irq_data.affinity);
1f5a5b87 TG	129	}
	130	#else
	131	static inline void free_masks(struct irq_desc *desc) { }
	132	#endif
	133
b6873807	134	static struct irq_desc alloc_desc(int irq, int node, struct module owner)
1f5a5b87 TG	135	{
1f5a5b87 TG	136	struct irq_desc *desc;
baa0d233	137	gfp_t gfp = GFP_KERNEL;
1f5a5b87 TG	138
	139	desc = kzalloc_node(sizeof(*desc), gfp, node);
	140	if (!desc)
	141	return NULL;
	142	/* allocate based on nr_cpu_ids */
6c9ae009	143	desc->kstat_irqs = alloc_percpu(unsigned int);
1f5a5b87 TG	144	if (!desc->kstat_irqs)
	145	goto err_desc;
	146
	147	if (alloc_masks(desc, gfp, node))
	148	goto err_kstat;
	149
	150	raw_spin_lock_init(&desc->lock);
	151	lockdep_set_class(&desc->lock, &irq_desc_lock_class);
	152
b6873807	153	desc_set_defaults(irq, desc, node, owner);
1f5a5b87 TG	154
	155	return desc;
	156
	157	err_kstat:
6c9ae009	158	free_percpu(desc->kstat_irqs);
1f5a5b87 TG	159	err_desc:
	160	kfree(desc);
	161	return NULL;
	162	}
	163
	164	static void free_desc(unsigned int irq)
	165	{
	166	struct irq_desc *desc = irq_to_desc(irq);
1f5a5b87	167
13bfe99e TG	168	unregister_irq_proc(irq, desc);
13bfe99e TG	169
a05a900a	170	mutex_lock(&sparse_irq_lock);
1f5a5b87	171	delete_irq_desc(irq);
a05a900a	172	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	173
1f5a5b87 TG	174	free_masks(desc);
6c9ae009	175	free_percpu(desc->kstat_irqs);
1f5a5b87 TG	176	kfree(desc);
	177	}
	178
b6873807 SAS	179	static int alloc_descs(unsigned int start, unsigned int cnt, int node,
b6873807 SAS	180	struct module *owner)
1f5a5b87 TG	181	{
1f5a5b87 TG	182	struct irq_desc *desc;
1f5a5b87 TG	183	int i;
	184
	185	for (i = 0; i < cnt; i++) {
b6873807	186	desc = alloc_desc(start + i, node, owner);
1f5a5b87 TG	187	if (!desc)
1f5a5b87 TG	188	goto err;
a05a900a	189	mutex_lock(&sparse_irq_lock);
1f5a5b87	190	irq_insert_desc(start + i, desc);
a05a900a	191	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	192	}
	193	return start;
	194
	195	err:
	196	for (i--; i >= 0; i--)
	197	free_desc(start + i);
	198
a05a900a	199	mutex_lock(&sparse_irq_lock);
1f5a5b87	200	bitmap_clear(allocated_irqs, start, cnt);
a05a900a	201	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	202	return -ENOMEM;
	203	}
	204
ed4dea6e	205	static int irq_expand_nr_irqs(unsigned int nr)
e7bcecb7	206	{
ed4dea6e	207	if (nr > IRQ_BITMAP_BITS)
e7bcecb7	208	return -ENOMEM;
ed4dea6e	209	nr_irqs = nr;
e7bcecb7 TG	210	return 0;
	211	}
	212
3795de23 TG	213	int __init early_irq_init(void)
3795de23 TG	214	{
b683de2b	215	int i, initcnt, node = first_online_node;
3795de23	216	struct irq_desc *desc;
3795de23 TG	217
	218	init_irq_default_affinity();
	219
b683de2b TG	220	/* Let arch update nr_irqs and return the nr of preallocated irqs */
	221	initcnt = arch_probe_nr_irqs();
	222	printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt);
3795de23	223
c1ee6264 TG	224	if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
	225	nr_irqs = IRQ_BITMAP_BITS;
	226
	227	if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
	228	initcnt = IRQ_BITMAP_BITS;
	229
	230	if (initcnt > nr_irqs)
	231	nr_irqs = initcnt;
	232
b683de2b	233	for (i = 0; i < initcnt; i++) {
b6873807	234	desc = alloc_desc(i, node, NULL);
aa99ec0f TG	235	set_bit(i, allocated_irqs);
aa99ec0f TG	236	irq_insert_desc(i, desc);
3795de23	237	}
3795de23 TG	238	return arch_early_irq_init();
	239	}
	240
3795de23 TG	241	#else /* !CONFIG_SPARSE_IRQ */
	242
	243	struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
	244	[0 ... NR_IRQS-1] = {
3795de23 TG	245	.handle_irq = handle_bad_irq,
	246	.depth = 1,
	247	.lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
	248	}
	249	};
	250
3795de23 TG	251	int __init early_irq_init(void)
3795de23 TG	252	{
aa99ec0f	253	int count, i, node = first_online_node;
3795de23	254	struct irq_desc *desc;
3795de23 TG	255
	256	init_irq_default_affinity();
	257
	258	printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);
	259
	260	desc = irq_desc;
	261	count = ARRAY_SIZE(irq_desc);
	262
	263	for (i = 0; i < count; i++) {
6c9ae009	264	desc[i].kstat_irqs = alloc_percpu(unsigned int);
e7fbad30 LW	265	alloc_masks(&desc[i], GFP_KERNEL, node);
e7fbad30 LW	266	raw_spin_lock_init(&desc[i].lock);
154cd387	267	lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
b6873807	268	desc_set_defaults(i, &desc[i], node, NULL);
3795de23 TG	269	}
	270	return arch_early_irq_init();
	271	}
	272
	273	struct irq_desc *irq_to_desc(unsigned int irq)
	274	{
	275	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
	276	}
2c45aada	277	EXPORT_SYMBOL(irq_to_desc);
3795de23	278
1f5a5b87 TG	279	static void free_desc(unsigned int irq)
1f5a5b87 TG	280	{
b7b29338	281	dynamic_irq_cleanup(irq);
1f5a5b87 TG	282	}
1f5a5b87 TG	283
b6873807 SAS	284	static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
b6873807 SAS	285	struct module *owner)
1f5a5b87	286	{
b6873807 SAS	287	u32 i;
	288
	289	for (i = 0; i < cnt; i++) {
	290	struct irq_desc *desc = irq_to_desc(start + i);
	291
	292	desc->owner = owner;
	293	}
1f5a5b87 TG	294	return start;
1f5a5b87 TG	295	}
e7bcecb7	296
ed4dea6e	297	static int irq_expand_nr_irqs(unsigned int nr)
e7bcecb7 TG	298	{
	299	return -ENOMEM;
	300	}
	301
3795de23 TG	302	#endif /* !CONFIG_SPARSE_IRQ */
3795de23 TG	303
fe12bc2c TG	304	/**
	305	* generic_handle_irq - Invoke the handler for a particular irq
	306	* @irq: The irq number to handle
	307	*
	308	*/
	309	int generic_handle_irq(unsigned int irq)
	310	{
	311	struct irq_desc *desc = irq_to_desc(irq);
	312
	313	if (!desc)
	314	return -EINVAL;
	315	generic_handle_irq_desc(irq, desc);
	316	return 0;
	317	}
edf76f83	318	EXPORT_SYMBOL_GPL(generic_handle_irq);
fe12bc2c	319
1f5a5b87 TG	320	/* Dynamic interrupt handling */
	321
	322	/**
	323	* irq_free_descs - free irq descriptors
	324	* @from: Start of descriptor range
	325	* @cnt: Number of consecutive irqs to free
	326	*/
	327	void irq_free_descs(unsigned int from, unsigned int cnt)
	328	{
1f5a5b87 TG	329	int i;
	330
	331	if (from >= nr_irqs \|\| (from + cnt) > nr_irqs)
	332	return;
	333
	334	for (i = 0; i < cnt; i++)
	335	free_desc(from + i);
	336
a05a900a	337	mutex_lock(&sparse_irq_lock);
1f5a5b87	338	bitmap_clear(allocated_irqs, from, cnt);
a05a900a	339	mutex_unlock(&sparse_irq_lock);
1f5a5b87	340	}
edf76f83	341	EXPORT_SYMBOL_GPL(irq_free_descs);
1f5a5b87 TG	342
	343	/**
	344	* irq_alloc_descs - allocate and initialize a range of irq descriptors
	345	* @irq: Allocate for specific irq number if irq >= 0
	346	* @from: Start the search from this irq number
	347	* @cnt: Number of consecutive irqs to allocate.
	348	* @node: Preferred node on which the irq descriptor should be allocated
d522a0d1	349	* @owner: Owning module (can be NULL)
1f5a5b87 TG	350	*
	351	* Returns the first irq number or error code
	352	*/
	353	int __ref
b6873807 SAS	354	__irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
b6873807 SAS	355	struct module *owner)
1f5a5b87	356	{
1f5a5b87 TG	357	int start, ret;
	358
	359	if (!cnt)
	360	return -EINVAL;
	361
c5182b88 MB	362	if (irq >= 0) {
	363	if (from > irq)
	364	return -EINVAL;
	365	from = irq;
	366	}
	367
a05a900a	368	mutex_lock(&sparse_irq_lock);
1f5a5b87	369
ed4dea6e YL	370	start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
ed4dea6e YL	371	from, cnt, 0);
1f5a5b87 TG	372	ret = -EEXIST;
	373	if (irq >=0 && start != irq)
	374	goto err;
	375
ed4dea6e YL	376	if (start + cnt > nr_irqs) {
ed4dea6e YL	377	ret = irq_expand_nr_irqs(start + cnt);
e7bcecb7 TG	378	if (ret)
	379	goto err;
	380	}
1f5a5b87 TG	381
1f5a5b87 TG	382	bitmap_set(allocated_irqs, start, cnt);
a05a900a	383	mutex_unlock(&sparse_irq_lock);
b6873807	384	return alloc_descs(start, cnt, node, owner);
1f5a5b87 TG	385
1f5a5b87 TG	386	err:
a05a900a	387	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	388	return ret;
1f5a5b87 TG	389	}
b6873807	390	EXPORT_SYMBOL_GPL(__irq_alloc_descs);
1f5a5b87	391
06f6c339 TG	392	/**
	393	* irq_reserve_irqs - mark irqs allocated
	394	* @from: mark from irq number
	395	* @cnt: number of irqs to mark
	396	*
	397	* Returns 0 on success or an appropriate error code
	398	*/
	399	int irq_reserve_irqs(unsigned int from, unsigned int cnt)
	400	{
06f6c339 TG	401	unsigned int start;
	402	int ret = 0;
	403
	404	if (!cnt \|\| (from + cnt) > nr_irqs)
	405	return -EINVAL;
	406
a05a900a	407	mutex_lock(&sparse_irq_lock);
06f6c339 TG	408	start = bitmap_find_next_zero_area(allocated_irqs, nr_irqs, from, cnt, 0);
	409	if (start == from)
	410	bitmap_set(allocated_irqs, start, cnt);
	411	else
	412	ret = -EEXIST;
a05a900a	413	mutex_unlock(&sparse_irq_lock);
06f6c339 TG	414	return ret;
	415	}
	416
a98d24b7 TG	417	/**
	418	* irq_get_next_irq - get next allocated irq number
	419	* @offset: where to start the search
	420	*
	421	* Returns next irq number after offset or nr_irqs if none is found.
	422	*/
	423	unsigned int irq_get_next_irq(unsigned int offset)
	424	{
	425	return find_next_bit(allocated_irqs, nr_irqs, offset);
	426	}
	427
d5eb4ad2	428	struct irq_desc *
31d9d9b6 MZ	429	__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
31d9d9b6 MZ	430	unsigned int check)
d5eb4ad2 TG	431	{
	432	struct irq_desc *desc = irq_to_desc(irq);
	433
	434	if (desc) {
31d9d9b6 MZ	435	if (check & _IRQ_DESC_CHECK) {
	436	if ((check & _IRQ_DESC_PERCPU) &&
	437	!irq_settings_is_per_cpu_devid(desc))
	438	return NULL;
	439
	440	if (!(check & _IRQ_DESC_PERCPU) &&
	441	irq_settings_is_per_cpu_devid(desc))
	442	return NULL;
	443	}
	444
d5eb4ad2 TG	445	if (bus)
	446	chip_bus_lock(desc);
	447	raw_spin_lock_irqsave(&desc->lock, *flags);
	448	}
	449	return desc;
	450	}
	451
	452	void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
	453	{
	454	raw_spin_unlock_irqrestore(&desc->lock, flags);
	455	if (bus)
	456	chip_bus_sync_unlock(desc);
	457	}
	458
31d9d9b6 MZ	459	int irq_set_percpu_devid(unsigned int irq)
	460	{
	461	struct irq_desc *desc = irq_to_desc(irq);
	462
	463	if (!desc)
	464	return -EINVAL;
	465
	466	if (desc->percpu_enabled)
	467	return -EINVAL;
	468
	469	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
	470
	471	if (!desc->percpu_enabled)
	472	return -ENOMEM;
	473
	474	irq_set_percpu_devid_flags(irq);
	475	return 0;
	476	}
	477
b7b29338 TG	478	/**
	479	* dynamic_irq_cleanup - cleanup a dynamically allocated irq
	480	* @irq: irq number to initialize
	481	*/
	482	void dynamic_irq_cleanup(unsigned int irq)
3795de23	483	{
b7b29338 TG	484	struct irq_desc *desc = irq_to_desc(irq);
	485	unsigned long flags;
	486
	487	raw_spin_lock_irqsave(&desc->lock, flags);
b6873807	488	desc_set_defaults(irq, desc, desc_node(desc), NULL);
b7b29338	489	raw_spin_unlock_irqrestore(&desc->lock, flags);
3795de23 TG	490	}
3795de23 TG	491
792d0018 TG	492	void kstat_incr_irq_this_cpu(unsigned int irq)
	493	{
	494	kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
	495	}
	496
3795de23 TG	497	unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
	498	{
	499	struct irq_desc *desc = irq_to_desc(irq);
6c9ae009 ED	500
	501	return desc && desc->kstat_irqs ?
	502	*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
3795de23	503	}
478735e3	504
478735e3 KH	505	unsigned int kstat_irqs(unsigned int irq)
	506	{
	507	struct irq_desc *desc = irq_to_desc(irq);
	508	int cpu;
	509	int sum = 0;
	510
6c9ae009	511	if (!desc \|\| !desc->kstat_irqs)
478735e3 KH	512	return 0;
478735e3 KH	513	for_each_possible_cpu(cpu)
6c9ae009	514	sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
478735e3 KH	515	return sum;
478735e3 KH	516	}