[deliverable/linux.git] / kernel / irq / irqdesc.c

/*
 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
 * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
 *
 * This file contains the interrupt descriptor management code
 *
 * Detailed information is available in Documentation/DocBook/genericirq
 *
 */
#include <linux/irq.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/radix-tree.h>
#include <linux/bitmap.h>
#include <linux/irqdomain.h>

#include "internals.h"

/*
 * lockdep: we want to handle all irq_desc locks as a single lock-class:
 */
static struct lock_class_key irq_desc_lock_class;

#if defined(CONFIG_SMP)
static void __init init_irq_default_affinity(void)
{
	alloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
	cpumask_setall(irq_default_affinity);
}
#else
static void __init init_irq_default_affinity(void)
{
}
#endif

#ifdef CONFIG_SMP
static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
{
	if (!zalloc_cpumask_var_node(&desc->irq_data.affinity, gfp, node))
		return -ENOMEM;

#ifdef CONFIG_GENERIC_PENDING_IRQ
	if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
		free_cpumask_var(desc->irq_data.affinity);
		return -ENOMEM;
	}
#endif
	return 0;
}

static void desc_smp_init(struct irq_desc *desc, int node)
{
	desc->irq_data.node = node;
	cpumask_copy(desc->irq_data.affinity, irq_default_affinity);
#ifdef CONFIG_GENERIC_PENDING_IRQ
	cpumask_clear(desc->pending_mask);
#endif
}

static inline int desc_node(struct irq_desc *desc)
{
	return desc->irq_data.node;
}

#else
static inline int
alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; }
static inline void desc_smp_init(struct irq_desc *desc, int node) { }
static inline int desc_node(struct irq_desc *desc) { return 0; }
#endif

static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
		struct module *owner)
{
	int cpu;

	desc->irq_data.irq = irq;
	desc->irq_data.chip = &no_irq_chip;
	desc->irq_data.chip_data = NULL;
	desc->irq_data.handler_data = NULL;
	desc->irq_data.msi_desc = NULL;
	irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
	desc->handle_irq = handle_bad_irq;
	desc->depth = 1;
	desc->irq_count = 0;
	desc->irqs_unhandled = 0;
	desc->name = NULL;
	desc->owner = owner;
	for_each_possible_cpu(cpu)
		*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
	desc_smp_init(desc, node);
}

int nr_irqs = NR_IRQS;
EXPORT_SYMBOL_GPL(nr_irqs);

static DEFINE_MUTEX(sparse_irq_lock);
static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);

#ifdef CONFIG_SPARSE_IRQ

static RADIX_TREE(irq_desc_tree, GFP_KERNEL);

static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
{
	radix_tree_insert(&irq_desc_tree, irq, desc);
}

struct irq_desc *irq_to_desc(unsigned int irq)
{
	return radix_tree_lookup(&irq_desc_tree, irq);
}
EXPORT_SYMBOL(irq_to_desc);

static void delete_irq_desc(unsigned int irq)
{
	radix_tree_delete(&irq_desc_tree, irq);
}

#ifdef CONFIG_SMP
static void free_masks(struct irq_desc *desc)
{
#ifdef CONFIG_GENERIC_PENDING_IRQ
	free_cpumask_var(desc->pending_mask);
#endif
	free_cpumask_var(desc->irq_data.affinity);
}
#else
static inline void free_masks(struct irq_desc *desc) { }
#endif

static struct irq_desc *alloc_desc(int irq, int node, struct module *owner)
{
	struct irq_desc *desc;
	gfp_t gfp = GFP_KERNEL;

	desc = kzalloc_node(sizeof(*desc), gfp, node);
	if (!desc)
		return NULL;
	/* allocate based on nr_cpu_ids */
	desc->kstat_irqs = alloc_percpu(unsigned int);
	if (!desc->kstat_irqs)
		goto err_desc;

	if (alloc_masks(desc, gfp, node))
		goto err_kstat;

	raw_spin_lock_init(&desc->lock);
	lockdep_set_class(&desc->lock, &irq_desc_lock_class);

	desc_set_defaults(irq, desc, node, owner);

	return desc;

err_kstat:
	free_percpu(desc->kstat_irqs);
err_desc:
	kfree(desc);
	return NULL;
}

static void free_desc(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	unregister_irq_proc(irq, desc);

	mutex_lock(&sparse_irq_lock);
	delete_irq_desc(irq);
	mutex_unlock(&sparse_irq_lock);

	free_masks(desc);
	free_percpu(desc->kstat_irqs);
	kfree(desc);
}

static int alloc_descs(unsigned int start, unsigned int cnt, int node,
		       struct module *owner)
{
	struct irq_desc *desc;
	int i;

	for (i = 0; i < cnt; i++) {
		desc = alloc_desc(start + i, node, owner);
		if (!desc)
			goto err;
		mutex_lock(&sparse_irq_lock);
		irq_insert_desc(start + i, desc);
		mutex_unlock(&sparse_irq_lock);
	}
	return start;

err:
	for (i--; i >= 0; i--)
		free_desc(start + i);

	mutex_lock(&sparse_irq_lock);
	bitmap_clear(allocated_irqs, start, cnt);
	mutex_unlock(&sparse_irq_lock);
	return -ENOMEM;
}

static int irq_expand_nr_irqs(unsigned int nr)
{
	if (nr > IRQ_BITMAP_BITS)
		return -ENOMEM;
	nr_irqs = nr;
	return 0;
}

int __init early_irq_init(void)
{
	int i, initcnt, node = first_online_node;
	struct irq_desc *desc;

	init_irq_default_affinity();

	/* Let arch update nr_irqs and return the nr of preallocated irqs */
	initcnt = arch_probe_nr_irqs();
	printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt);

	if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
		nr_irqs = IRQ_BITMAP_BITS;

	if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
		initcnt = IRQ_BITMAP_BITS;

	if (initcnt > nr_irqs)
		nr_irqs = initcnt;

	for (i = 0; i < initcnt; i++) {
		desc = alloc_desc(i, node, NULL);
		set_bit(i, allocated_irqs);
		irq_insert_desc(i, desc);
	}
	return arch_early_irq_init();
}

#else /* !CONFIG_SPARSE_IRQ */

struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
	[0 ... NR_IRQS-1] = {
		.handle_irq	= handle_bad_irq,
		.depth		= 1,
		.lock		= __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
	}
};

int __init early_irq_init(void)
{
	int count, i, node = first_online_node;
	struct irq_desc *desc;

	init_irq_default_affinity();

	printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);

	desc = irq_desc;
	count = ARRAY_SIZE(irq_desc);

	for (i = 0; i < count; i++) {
		desc[i].kstat_irqs = alloc_percpu(unsigned int);
		alloc_masks(&desc[i], GFP_KERNEL, node);
		raw_spin_lock_init(&desc[i].lock);
		lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
		desc_set_defaults(i, &desc[i], node, NULL);
	}
	return arch_early_irq_init();
}

struct irq_desc *irq_to_desc(unsigned int irq)
{
	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
}
EXPORT_SYMBOL(irq_to_desc);

static void free_desc(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);
	unsigned long flags;

	raw_spin_lock_irqsave(&desc->lock, flags);
	desc_set_defaults(irq, desc, desc_node(desc), NULL);
	raw_spin_unlock_irqrestore(&desc->lock, flags);
}

static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
			      struct module *owner)
{
	u32 i;

	for (i = 0; i < cnt; i++) {
		struct irq_desc *desc = irq_to_desc(start + i);

		desc->owner = owner;
	}
	return start;
}

static int irq_expand_nr_irqs(unsigned int nr)
{
	return -ENOMEM;
}

void irq_mark_irq(unsigned int irq)
{
	mutex_lock(&sparse_irq_lock);
	bitmap_set(allocated_irqs, irq, 1);
	mutex_unlock(&sparse_irq_lock);
}

#ifdef CONFIG_GENERIC_IRQ_LEGACY
void irq_init_desc(unsigned int irq)
{
	free_desc(irq);
}
#endif

#endif /* !CONFIG_SPARSE_IRQ */

/**
 * generic_handle_irq - Invoke the handler for a particular irq
 * @irq:	The irq number to handle
 *
 */
int generic_handle_irq(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	if (!desc)
		return -EINVAL;
	generic_handle_irq_desc(irq, desc);
	return 0;
}
EXPORT_SYMBOL_GPL(generic_handle_irq);

#ifdef CONFIG_HANDLE_DOMAIN_IRQ
/**
 * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain
 * @domain:	The domain where to perform the lookup
 * @hwirq:	The HW irq number to convert to a logical one
 * @lookup:	Whether to perform the domain lookup or not
 * @regs:	Register file coming from the low-level handling code
 *
 * Returns:	0 on success, or -EINVAL if conversion has failed
 */
int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
			bool lookup, struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);
	unsigned int irq = hwirq;
	int ret = 0;

	irq_enter();

#ifdef CONFIG_IRQ_DOMAIN
	if (lookup)
		irq = irq_find_mapping(domain, hwirq);
#endif

	/*
	 * Some hardware gives randomly wrong interrupts.  Rather
	 * than crashing, do something sensible.
	 */
	if (unlikely(!irq || irq >= nr_irqs)) {
		ack_bad_irq(irq);
		ret = -EINVAL;
	} else {
		generic_handle_irq(irq);
	}

	irq_exit();
	set_irq_regs(old_regs);
	return ret;
}
#endif

/* Dynamic interrupt handling */

/**
 * irq_free_descs - free irq descriptors
 * @from:	Start of descriptor range
 * @cnt:	Number of consecutive irqs to free
 */
void irq_free_descs(unsigned int from, unsigned int cnt)
{
	int i;

	if (from >= nr_irqs || (from + cnt) > nr_irqs)
		return;

	for (i = 0; i < cnt; i++)
		free_desc(from + i);

	mutex_lock(&sparse_irq_lock);
	bitmap_clear(allocated_irqs, from, cnt);
	mutex_unlock(&sparse_irq_lock);
}
EXPORT_SYMBOL_GPL(irq_free_descs);

/**
 * irq_alloc_descs - allocate and initialize a range of irq descriptors
 * @irq:	Allocate for specific irq number if irq >= 0
 * @from:	Start the search from this irq number
 * @cnt:	Number of consecutive irqs to allocate.
 * @node:	Preferred node on which the irq descriptor should be allocated
 * @owner:	Owning module (can be NULL)
 *
 * Returns the first irq number or error code
 */
int __ref
__irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
		  struct module *owner)
{
	int start, ret;

	if (!cnt)
		return -EINVAL;

	if (irq >= 0) {
		if (from > irq)
			return -EINVAL;
		from = irq;
	} else {
		/*
		 * For interrupts which are freely allocated the
		 * architecture can force a lower bound to the @from
		 * argument. x86 uses this to exclude the GSI space.
		 */
		from = arch_dynirq_lower_bound(from);
	}

	mutex_lock(&sparse_irq_lock);

	start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
					   from, cnt, 0);
	ret = -EEXIST;
	if (irq >=0 && start != irq)
		goto err;

	if (start + cnt > nr_irqs) {
		ret = irq_expand_nr_irqs(start + cnt);
		if (ret)
			goto err;
	}

	bitmap_set(allocated_irqs, start, cnt);
	mutex_unlock(&sparse_irq_lock);
	return alloc_descs(start, cnt, node, owner);

err:
	mutex_unlock(&sparse_irq_lock);
	return ret;
}
EXPORT_SYMBOL_GPL(__irq_alloc_descs);

#ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
/**
 * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware
 * @cnt:	number of interrupts to allocate
 * @node:	node on which to allocate
 *
 * Returns an interrupt number > 0 or 0, if the allocation fails.
 */
unsigned int irq_alloc_hwirqs(int cnt, int node)
{
	int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL);

	if (irq < 0)
		return 0;

	for (i = irq; cnt > 0; i++, cnt--) {
		if (arch_setup_hwirq(i, node))
			goto err;
		irq_clear_status_flags(i, _IRQ_NOREQUEST);
	}
	return irq;

err:
	for (i--; i >= irq; i--) {
		irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
		arch_teardown_hwirq(i);
	}
	irq_free_descs(irq, cnt);
	return 0;
}
EXPORT_SYMBOL_GPL(irq_alloc_hwirqs);

/**
 * irq_free_hwirqs - Free irq descriptor and cleanup the hardware
 * @from:	Free from irq number
 * @cnt:	number of interrupts to free
 *
 */
void irq_free_hwirqs(unsigned int from, int cnt)
{
	int i, j;

	for (i = from, j = cnt; j > 0; i++, j--) {
		irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
		arch_teardown_hwirq(i);
	}
	irq_free_descs(from, cnt);
}
EXPORT_SYMBOL_GPL(irq_free_hwirqs);
#endif

/**
 * irq_get_next_irq - get next allocated irq number
 * @offset:	where to start the search
 *
 * Returns next irq number after offset or nr_irqs if none is found.
 */
unsigned int irq_get_next_irq(unsigned int offset)
{
	return find_next_bit(allocated_irqs, nr_irqs, offset);
}

struct irq_desc *
__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
		    unsigned int check)
{
	struct irq_desc *desc = irq_to_desc(irq);

	if (desc) {
		if (check & _IRQ_DESC_CHECK) {
			if ((check & _IRQ_DESC_PERCPU) &&
			    !irq_settings_is_per_cpu_devid(desc))
				return NULL;

			if (!(check & _IRQ_DESC_PERCPU) &&
			    irq_settings_is_per_cpu_devid(desc))
				return NULL;
		}

		if (bus)
			chip_bus_lock(desc);
		raw_spin_lock_irqsave(&desc->lock, *flags);
	}
	return desc;
}

void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
{
	raw_spin_unlock_irqrestore(&desc->lock, flags);
	if (bus)
		chip_bus_sync_unlock(desc);
}

int irq_set_percpu_devid(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	if (!desc)
		return -EINVAL;

	if (desc->percpu_enabled)
		return -EINVAL;

	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);

	if (!desc->percpu_enabled)
		return -ENOMEM;

	irq_set_percpu_devid_flags(irq);
	return 0;
}

void kstat_incr_irq_this_cpu(unsigned int irq)
{
	kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
}

unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
{
	struct irq_desc *desc = irq_to_desc(irq);

	return desc && desc->kstat_irqs ?
			*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
}

unsigned int kstat_irqs(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);
	int cpu;
	int sum = 0;

	if (!desc || !desc->kstat_irqs)
		return 0;
	for_each_possible_cpu(cpu)
		sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
	return sum;
}
Commit	Line	Data
3795de23 TG	1	/*
	2	* Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
	3	* Copyright (C) 2005-2006, Thomas Gleixner, Russell King
	4	*
	5	* This file contains the interrupt descriptor management code
	6	*
	7	* Detailed information is available in Documentation/DocBook/genericirq
	8	*
	9	*/
	10	#include <linux/irq.h>
	11	#include <linux/slab.h>
ec53cf23	12	#include <linux/export.h>
3795de23 TG	13	#include <linux/interrupt.h>
	14	#include <linux/kernel_stat.h>
	15	#include <linux/radix-tree.h>
1f5a5b87	16	#include <linux/bitmap.h>
76ba59f8	17	#include <linux/irqdomain.h>
3795de23 TG	18
	19	#include "internals.h"
	20
	21	/*
	22	* lockdep: we want to handle all irq_desc locks as a single lock-class:
	23	*/
78f90d91	24	static struct lock_class_key irq_desc_lock_class;
3795de23	25
fe051434	26	#if defined(CONFIG_SMP)
3795de23 TG	27	static void __init init_irq_default_affinity(void)
	28	{
	29	alloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
	30	cpumask_setall(irq_default_affinity);
	31	}
	32	#else
	33	static void __init init_irq_default_affinity(void)
	34	{
	35	}
	36	#endif
	37
1f5a5b87 TG	38	#ifdef CONFIG_SMP
	39	static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node)
	40	{
	41	if (!zalloc_cpumask_var_node(&desc->irq_data.affinity, gfp, node))
	42	return -ENOMEM;
	43
	44	#ifdef CONFIG_GENERIC_PENDING_IRQ
	45	if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
	46	free_cpumask_var(desc->irq_data.affinity);
	47	return -ENOMEM;
	48	}
	49	#endif
	50	return 0;
	51	}
	52
	53	static void desc_smp_init(struct irq_desc *desc, int node)
	54	{
aa99ec0f	55	desc->irq_data.node = node;
1f5a5b87	56	cpumask_copy(desc->irq_data.affinity, irq_default_affinity);
b7b29338 TG	57	#ifdef CONFIG_GENERIC_PENDING_IRQ
	58	cpumask_clear(desc->pending_mask);
	59	#endif
	60	}
	61
	62	static inline int desc_node(struct irq_desc *desc)
	63	{
	64	return desc->irq_data.node;
1f5a5b87 TG	65	}
	66
	67	#else
	68	static inline int
	69	alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; }
	70	static inline void desc_smp_init(struct irq_desc *desc, int node) { }
b7b29338	71	static inline int desc_node(struct irq_desc *desc) { return 0; }
1f5a5b87 TG	72	#endif
1f5a5b87 TG	73
b6873807 SAS	74	static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
b6873807 SAS	75	struct module *owner)
1f5a5b87	76	{
6c9ae009 ED	77	int cpu;
6c9ae009 ED	78
1f5a5b87 TG	79	desc->irq_data.irq = irq;
	80	desc->irq_data.chip = &no_irq_chip;
	81	desc->irq_data.chip_data = NULL;
	82	desc->irq_data.handler_data = NULL;
	83	desc->irq_data.msi_desc = NULL;
f9e4989e	84	irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
801a0e9a	85	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
1f5a5b87 TG	86	desc->handle_irq = handle_bad_irq;
1f5a5b87 TG	87	desc->depth = 1;
b7b29338 TG	88	desc->irq_count = 0;
b7b29338 TG	89	desc->irqs_unhandled = 0;
1f5a5b87	90	desc->name = NULL;
b6873807	91	desc->owner = owner;
6c9ae009 ED	92	for_each_possible_cpu(cpu)
6c9ae009 ED	93	*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
1f5a5b87 TG	94	desc_smp_init(desc, node);
	95	}
	96
3795de23 TG	97	int nr_irqs = NR_IRQS;
	98	EXPORT_SYMBOL_GPL(nr_irqs);
	99
a05a900a	100	static DEFINE_MUTEX(sparse_irq_lock);
c1ee6264	101	static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);
1f5a5b87	102
3795de23 TG	103	#ifdef CONFIG_SPARSE_IRQ
3795de23 TG	104
baa0d233	105	static RADIX_TREE(irq_desc_tree, GFP_KERNEL);
3795de23	106
1f5a5b87	107	static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
3795de23 TG	108	{
	109	radix_tree_insert(&irq_desc_tree, irq, desc);
	110	}
	111
	112	struct irq_desc *irq_to_desc(unsigned int irq)
	113	{
	114	return radix_tree_lookup(&irq_desc_tree, irq);
	115	}
3911ff30	116	EXPORT_SYMBOL(irq_to_desc);
3795de23	117
1f5a5b87 TG	118	static void delete_irq_desc(unsigned int irq)
	119	{
	120	radix_tree_delete(&irq_desc_tree, irq);
	121	}
	122
	123	#ifdef CONFIG_SMP
	124	static void free_masks(struct irq_desc *desc)
	125	{
	126	#ifdef CONFIG_GENERIC_PENDING_IRQ
	127	free_cpumask_var(desc->pending_mask);
	128	#endif
c0a19ebc	129	free_cpumask_var(desc->irq_data.affinity);
1f5a5b87 TG	130	}
	131	#else
	132	static inline void free_masks(struct irq_desc *desc) { }
	133	#endif
	134
b6873807	135	static struct irq_desc alloc_desc(int irq, int node, struct module owner)
1f5a5b87 TG	136	{
1f5a5b87 TG	137	struct irq_desc *desc;
baa0d233	138	gfp_t gfp = GFP_KERNEL;
1f5a5b87 TG	139
	140	desc = kzalloc_node(sizeof(*desc), gfp, node);
	141	if (!desc)
	142	return NULL;
	143	/* allocate based on nr_cpu_ids */
6c9ae009	144	desc->kstat_irqs = alloc_percpu(unsigned int);
1f5a5b87 TG	145	if (!desc->kstat_irqs)
	146	goto err_desc;
	147
	148	if (alloc_masks(desc, gfp, node))
	149	goto err_kstat;
	150
	151	raw_spin_lock_init(&desc->lock);
	152	lockdep_set_class(&desc->lock, &irq_desc_lock_class);
	153
b6873807	154	desc_set_defaults(irq, desc, node, owner);
1f5a5b87 TG	155
	156	return desc;
	157
	158	err_kstat:
6c9ae009	159	free_percpu(desc->kstat_irqs);
1f5a5b87 TG	160	err_desc:
	161	kfree(desc);
	162	return NULL;
	163	}
	164
	165	static void free_desc(unsigned int irq)
	166	{
	167	struct irq_desc *desc = irq_to_desc(irq);
1f5a5b87	168
13bfe99e TG	169	unregister_irq_proc(irq, desc);
13bfe99e TG	170
a05a900a	171	mutex_lock(&sparse_irq_lock);
1f5a5b87	172	delete_irq_desc(irq);
a05a900a	173	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	174
1f5a5b87 TG	175	free_masks(desc);
6c9ae009	176	free_percpu(desc->kstat_irqs);
1f5a5b87 TG	177	kfree(desc);
	178	}
	179
b6873807 SAS	180	static int alloc_descs(unsigned int start, unsigned int cnt, int node,
b6873807 SAS	181	struct module *owner)
1f5a5b87 TG	182	{
1f5a5b87 TG	183	struct irq_desc *desc;
1f5a5b87 TG	184	int i;
	185
	186	for (i = 0; i < cnt; i++) {
b6873807	187	desc = alloc_desc(start + i, node, owner);
1f5a5b87 TG	188	if (!desc)
1f5a5b87 TG	189	goto err;
a05a900a	190	mutex_lock(&sparse_irq_lock);
1f5a5b87	191	irq_insert_desc(start + i, desc);
a05a900a	192	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	193	}
	194	return start;
	195
	196	err:
	197	for (i--; i >= 0; i--)
	198	free_desc(start + i);
	199
a05a900a	200	mutex_lock(&sparse_irq_lock);
1f5a5b87	201	bitmap_clear(allocated_irqs, start, cnt);
a05a900a	202	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	203	return -ENOMEM;
	204	}
	205
ed4dea6e	206	static int irq_expand_nr_irqs(unsigned int nr)
e7bcecb7	207	{
ed4dea6e	208	if (nr > IRQ_BITMAP_BITS)
e7bcecb7	209	return -ENOMEM;
ed4dea6e	210	nr_irqs = nr;
e7bcecb7 TG	211	return 0;
	212	}
	213
3795de23 TG	214	int __init early_irq_init(void)
3795de23 TG	215	{
b683de2b	216	int i, initcnt, node = first_online_node;
3795de23	217	struct irq_desc *desc;
3795de23 TG	218
	219	init_irq_default_affinity();
	220
b683de2b TG	221	/* Let arch update nr_irqs and return the nr of preallocated irqs */
	222	initcnt = arch_probe_nr_irqs();
	223	printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt);
3795de23	224
c1ee6264 TG	225	if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
	226	nr_irqs = IRQ_BITMAP_BITS;
	227
	228	if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
	229	initcnt = IRQ_BITMAP_BITS;
	230
	231	if (initcnt > nr_irqs)
	232	nr_irqs = initcnt;
	233
b683de2b	234	for (i = 0; i < initcnt; i++) {
b6873807	235	desc = alloc_desc(i, node, NULL);
aa99ec0f TG	236	set_bit(i, allocated_irqs);
aa99ec0f TG	237	irq_insert_desc(i, desc);
3795de23	238	}
3795de23 TG	239	return arch_early_irq_init();
	240	}
	241
3795de23 TG	242	#else /* !CONFIG_SPARSE_IRQ */
	243
	244	struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
	245	[0 ... NR_IRQS-1] = {
3795de23 TG	246	.handle_irq = handle_bad_irq,
	247	.depth = 1,
	248	.lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
	249	}
	250	};
	251
3795de23 TG	252	int __init early_irq_init(void)
3795de23 TG	253	{
aa99ec0f	254	int count, i, node = first_online_node;
3795de23	255	struct irq_desc *desc;
3795de23 TG	256
	257	init_irq_default_affinity();
	258
	259	printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);
	260
	261	desc = irq_desc;
	262	count = ARRAY_SIZE(irq_desc);
	263
	264	for (i = 0; i < count; i++) {
6c9ae009	265	desc[i].kstat_irqs = alloc_percpu(unsigned int);
e7fbad30 LW	266	alloc_masks(&desc[i], GFP_KERNEL, node);
e7fbad30 LW	267	raw_spin_lock_init(&desc[i].lock);
154cd387	268	lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
b6873807	269	desc_set_defaults(i, &desc[i], node, NULL);
3795de23 TG	270	}
	271	return arch_early_irq_init();
	272	}
	273
	274	struct irq_desc *irq_to_desc(unsigned int irq)
	275	{
	276	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
	277	}
2c45aada	278	EXPORT_SYMBOL(irq_to_desc);
3795de23	279
1f5a5b87 TG	280	static void free_desc(unsigned int irq)
1f5a5b87 TG	281	{
d8179bc0 TG	282	struct irq_desc *desc = irq_to_desc(irq);
	283	unsigned long flags;
	284
	285	raw_spin_lock_irqsave(&desc->lock, flags);
	286	desc_set_defaults(irq, desc, desc_node(desc), NULL);
	287	raw_spin_unlock_irqrestore(&desc->lock, flags);
1f5a5b87 TG	288	}
1f5a5b87 TG	289
b6873807 SAS	290	static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
b6873807 SAS	291	struct module *owner)
1f5a5b87	292	{
b6873807 SAS	293	u32 i;
	294
	295	for (i = 0; i < cnt; i++) {
	296	struct irq_desc *desc = irq_to_desc(start + i);
	297
	298	desc->owner = owner;
	299	}
1f5a5b87 TG	300	return start;
1f5a5b87 TG	301	}
e7bcecb7	302
ed4dea6e	303	static int irq_expand_nr_irqs(unsigned int nr)
e7bcecb7 TG	304	{
	305	return -ENOMEM;
	306	}
	307
f63b6a05 TG	308	void irq_mark_irq(unsigned int irq)
	309	{
	310	mutex_lock(&sparse_irq_lock);
	311	bitmap_set(allocated_irqs, irq, 1);
	312	mutex_unlock(&sparse_irq_lock);
	313	}
	314
c940e01c TG	315	#ifdef CONFIG_GENERIC_IRQ_LEGACY
	316	void irq_init_desc(unsigned int irq)
	317	{
d8179bc0	318	free_desc(irq);
c940e01c TG	319	}
	320	#endif
	321
3795de23 TG	322	#endif /* !CONFIG_SPARSE_IRQ */
3795de23 TG	323
fe12bc2c TG	324	/**
	325	* generic_handle_irq - Invoke the handler for a particular irq
	326	* @irq: The irq number to handle
	327	*
	328	*/
	329	int generic_handle_irq(unsigned int irq)
	330	{
	331	struct irq_desc *desc = irq_to_desc(irq);
	332
	333	if (!desc)
	334	return -EINVAL;
	335	generic_handle_irq_desc(irq, desc);
	336	return 0;
	337	}
edf76f83	338	EXPORT_SYMBOL_GPL(generic_handle_irq);
fe12bc2c	339
76ba59f8 MZ	340	#ifdef CONFIG_HANDLE_DOMAIN_IRQ
	341	/**
	342	* __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain
	343	* @domain: The domain where to perform the lookup
	344	* @hwirq: The HW irq number to convert to a logical one
	345	* @lookup: Whether to perform the domain lookup or not
	346	* @regs: Register file coming from the low-level handling code
	347	*
	348	* Returns: 0 on success, or -EINVAL if conversion has failed
	349	*/
	350	int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
	351	bool lookup, struct pt_regs *regs)
	352	{
	353	struct pt_regs *old_regs = set_irq_regs(regs);
	354	unsigned int irq = hwirq;
	355	int ret = 0;
	356
	357	irq_enter();
	358
	359	#ifdef CONFIG_IRQ_DOMAIN
	360	if (lookup)
	361	irq = irq_find_mapping(domain, hwirq);
	362	#endif
	363
	364	/*
	365	* Some hardware gives randomly wrong interrupts. Rather
	366	* than crashing, do something sensible.
	367	*/
	368	if (unlikely(!irq \|\| irq >= nr_irqs)) {
	369	ack_bad_irq(irq);
	370	ret = -EINVAL;
	371	} else {
	372	generic_handle_irq(irq);
	373	}
	374
	375	irq_exit();
	376	set_irq_regs(old_regs);
	377	return ret;
	378	}
	379	#endif
	380
1f5a5b87 TG	381	/* Dynamic interrupt handling */
	382
	383	/**
	384	* irq_free_descs - free irq descriptors
	385	* @from: Start of descriptor range
	386	* @cnt: Number of consecutive irqs to free
	387	*/
	388	void irq_free_descs(unsigned int from, unsigned int cnt)
	389	{
1f5a5b87 TG	390	int i;
	391
	392	if (from >= nr_irqs \|\| (from + cnt) > nr_irqs)
	393	return;
	394
	395	for (i = 0; i < cnt; i++)
	396	free_desc(from + i);
	397
a05a900a	398	mutex_lock(&sparse_irq_lock);
1f5a5b87	399	bitmap_clear(allocated_irqs, from, cnt);
a05a900a	400	mutex_unlock(&sparse_irq_lock);
1f5a5b87	401	}
edf76f83	402	EXPORT_SYMBOL_GPL(irq_free_descs);
1f5a5b87 TG	403
	404	/**
	405	* irq_alloc_descs - allocate and initialize a range of irq descriptors
	406	* @irq: Allocate for specific irq number if irq >= 0
	407	* @from: Start the search from this irq number
	408	* @cnt: Number of consecutive irqs to allocate.
	409	* @node: Preferred node on which the irq descriptor should be allocated
d522a0d1	410	* @owner: Owning module (can be NULL)
1f5a5b87 TG	411	*
	412	* Returns the first irq number or error code
	413	*/
	414	int __ref
b6873807 SAS	415	__irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
b6873807 SAS	416	struct module *owner)
1f5a5b87	417	{
1f5a5b87 TG	418	int start, ret;
	419
	420	if (!cnt)
	421	return -EINVAL;
	422
c5182b88 MB	423	if (irq >= 0) {
	424	if (from > irq)
	425	return -EINVAL;
	426	from = irq;
62a08ae2 TG	427	} else {
	428	/*
	429	* For interrupts which are freely allocated the
	430	* architecture can force a lower bound to the @from
	431	* argument. x86 uses this to exclude the GSI space.
	432	*/
	433	from = arch_dynirq_lower_bound(from);
c5182b88 MB	434	}
c5182b88 MB	435
a05a900a	436	mutex_lock(&sparse_irq_lock);
1f5a5b87	437
ed4dea6e YL	438	start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
ed4dea6e YL	439	from, cnt, 0);
1f5a5b87 TG	440	ret = -EEXIST;
	441	if (irq >=0 && start != irq)
	442	goto err;
	443
ed4dea6e YL	444	if (start + cnt > nr_irqs) {
ed4dea6e YL	445	ret = irq_expand_nr_irqs(start + cnt);
e7bcecb7 TG	446	if (ret)
	447	goto err;
	448	}
1f5a5b87 TG	449
1f5a5b87 TG	450	bitmap_set(allocated_irqs, start, cnt);
a05a900a	451	mutex_unlock(&sparse_irq_lock);
b6873807	452	return alloc_descs(start, cnt, node, owner);
1f5a5b87 TG	453
1f5a5b87 TG	454	err:
a05a900a	455	mutex_unlock(&sparse_irq_lock);
1f5a5b87 TG	456	return ret;
1f5a5b87 TG	457	}
b6873807	458	EXPORT_SYMBOL_GPL(__irq_alloc_descs);
1f5a5b87	459
7b6ef126 TG	460	#ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
	461	/**
	462	* irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware
	463	* @cnt: number of interrupts to allocate
	464	* @node: node on which to allocate
	465	*
	466	* Returns an interrupt number > 0 or 0, if the allocation fails.
	467	*/
	468	unsigned int irq_alloc_hwirqs(int cnt, int node)
	469	{
	470	int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL);
	471
	472	if (irq < 0)
	473	return 0;
	474
	475	for (i = irq; cnt > 0; i++, cnt--) {
	476	if (arch_setup_hwirq(i, node))
	477	goto err;
	478	irq_clear_status_flags(i, _IRQ_NOREQUEST);
	479	}
	480	return irq;
	481
	482	err:
	483	for (i--; i >= irq; i--) {
	484	irq_set_status_flags(i, _IRQ_NOREQUEST \| _IRQ_NOPROBE);
	485	arch_teardown_hwirq(i);
	486	}
	487	irq_free_descs(irq, cnt);
	488	return 0;
	489	}
	490	EXPORT_SYMBOL_GPL(irq_alloc_hwirqs);
	491
	492	/**
	493	* irq_free_hwirqs - Free irq descriptor and cleanup the hardware
	494	* @from: Free from irq number
	495	* @cnt: number of interrupts to free
	496	*
	497	*/
	498	void irq_free_hwirqs(unsigned int from, int cnt)
	499	{
8844aad8	500	int i, j;
7b6ef126	501
8844aad8	502	for (i = from, j = cnt; j > 0; i++, j--) {
7b6ef126 TG	503	irq_set_status_flags(i, _IRQ_NOREQUEST \| _IRQ_NOPROBE);
	504	arch_teardown_hwirq(i);
	505	}
	506	irq_free_descs(from, cnt);
	507	}
	508	EXPORT_SYMBOL_GPL(irq_free_hwirqs);
	509	#endif
	510
a98d24b7 TG	511	/**
	512	* irq_get_next_irq - get next allocated irq number
	513	* @offset: where to start the search
	514	*
	515	* Returns next irq number after offset or nr_irqs if none is found.
	516	*/
	517	unsigned int irq_get_next_irq(unsigned int offset)
	518	{
	519	return find_next_bit(allocated_irqs, nr_irqs, offset);
	520	}
	521
d5eb4ad2	522	struct irq_desc *
31d9d9b6 MZ	523	__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
31d9d9b6 MZ	524	unsigned int check)
d5eb4ad2 TG	525	{
	526	struct irq_desc *desc = irq_to_desc(irq);
	527
	528	if (desc) {
31d9d9b6 MZ	529	if (check & _IRQ_DESC_CHECK) {
	530	if ((check & _IRQ_DESC_PERCPU) &&
	531	!irq_settings_is_per_cpu_devid(desc))
	532	return NULL;
	533
	534	if (!(check & _IRQ_DESC_PERCPU) &&
	535	irq_settings_is_per_cpu_devid(desc))
	536	return NULL;
	537	}
	538
d5eb4ad2 TG	539	if (bus)
	540	chip_bus_lock(desc);
	541	raw_spin_lock_irqsave(&desc->lock, *flags);
	542	}
	543	return desc;
	544	}
	545
	546	void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
	547	{
	548	raw_spin_unlock_irqrestore(&desc->lock, flags);
	549	if (bus)
	550	chip_bus_sync_unlock(desc);
	551	}
	552
31d9d9b6 MZ	553	int irq_set_percpu_devid(unsigned int irq)
	554	{
	555	struct irq_desc *desc = irq_to_desc(irq);
	556
	557	if (!desc)
	558	return -EINVAL;
	559
	560	if (desc->percpu_enabled)
	561	return -EINVAL;
	562
	563	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
	564
	565	if (!desc->percpu_enabled)
	566	return -ENOMEM;
	567
	568	irq_set_percpu_devid_flags(irq);
	569	return 0;
	570	}
	571
792d0018 TG	572	void kstat_incr_irq_this_cpu(unsigned int irq)
	573	{
	574	kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
	575	}
	576
3795de23 TG	577	unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
	578	{
	579	struct irq_desc *desc = irq_to_desc(irq);
6c9ae009 ED	580
	581	return desc && desc->kstat_irqs ?
	582	*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
3795de23	583	}
478735e3	584
478735e3 KH	585	unsigned int kstat_irqs(unsigned int irq)
	586	{
	587	struct irq_desc *desc = irq_to_desc(irq);
	588	int cpu;
	589	int sum = 0;
	590
6c9ae009	591	if (!desc \|\| !desc->kstat_irqs)
478735e3 KH	592	return 0;
478735e3 KH	593	for_each_possible_cpu(cpu)
6c9ae009	594	sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
478735e3 KH	595	return sum;
478735e3 KH	596	}