Commit | Line | Data |
---|---|---|
d17bf24e QY |
1 | /* |
2 | * linux/kernel/irq/ipi.c | |
3 | * | |
4 | * Copyright (C) 2015 Imagination Technologies Ltd | |
5 | * Author: Qais Yousef <qais.yousef@imgtec.com> | |
6 | * | |
7 | * This file contains driver APIs to the IPI subsystem. | |
8 | */ | |
9 | ||
10 | #define pr_fmt(fmt) "genirq/ipi: " fmt | |
11 | ||
12 | #include <linux/irqdomain.h> | |
13 | #include <linux/irq.h> | |
14 | ||
15 | /** | |
16 | * irq_reserve_ipi() - Setup an IPI to destination cpumask | |
17 | * @domain: IPI domain | |
18 | * @dest: cpumask of cpus which can receive the IPI | |
19 | * | |
20 | * Allocate a virq that can be used to send IPI to any CPU in dest mask. | |
21 | * | |
22 | * On success it'll return linux irq number and 0 on failure | |
23 | */ | |
24 | unsigned int irq_reserve_ipi(struct irq_domain *domain, | |
25 | const struct cpumask *dest) | |
26 | { | |
27 | unsigned int nr_irqs, offset; | |
28 | struct irq_data *data; | |
29 | int virq, i; | |
30 | ||
31 | if (!domain ||!irq_domain_is_ipi(domain)) { | |
32 | pr_warn("Reservation on a non IPI domain\n"); | |
33 | return 0; | |
34 | } | |
35 | ||
36 | if (!cpumask_subset(dest, cpu_possible_mask)) { | |
37 | pr_warn("Reservation is not in possible_cpu_mask\n"); | |
38 | return 0; | |
39 | } | |
40 | ||
41 | nr_irqs = cpumask_weight(dest); | |
42 | if (!nr_irqs) { | |
43 | pr_warn("Reservation for empty destination mask\n"); | |
44 | return 0; | |
45 | } | |
46 | ||
47 | if (irq_domain_is_ipi_single(domain)) { | |
48 | /* | |
49 | * If the underlying implementation uses a single HW irq on | |
50 | * all cpus then we only need a single Linux irq number for | |
51 | * it. We have no restrictions vs. the destination mask. The | |
52 | * underlying implementation can deal with holes nicely. | |
53 | */ | |
54 | nr_irqs = 1; | |
55 | offset = 0; | |
56 | } else { | |
57 | unsigned int next; | |
58 | ||
59 | /* | |
60 | * The IPI requires a seperate HW irq on each CPU. We require | |
61 | * that the destination mask is consecutive. If an | |
62 | * implementation needs to support holes, it can reserve | |
63 | * several IPI ranges. | |
64 | */ | |
65 | offset = cpumask_first(dest); | |
66 | /* | |
67 | * Find a hole and if found look for another set bit after the | |
68 | * hole. For now we don't support this scenario. | |
69 | */ | |
70 | next = cpumask_next_zero(offset, dest); | |
71 | if (next < nr_cpu_ids) | |
72 | next = cpumask_next(next, dest); | |
73 | if (next < nr_cpu_ids) { | |
74 | pr_warn("Destination mask has holes\n"); | |
75 | return 0; | |
76 | } | |
77 | } | |
78 | ||
79 | virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE); | |
80 | if (virq <= 0) { | |
81 | pr_warn("Can't reserve IPI, failed to alloc descs\n"); | |
82 | return 0; | |
83 | } | |
84 | ||
85 | virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE, | |
86 | (void *) dest, true); | |
87 | ||
88 | if (virq <= 0) { | |
89 | pr_warn("Can't reserve IPI, failed to alloc hw irqs\n"); | |
90 | goto free_descs; | |
91 | } | |
92 | ||
93 | for (i = 0; i < nr_irqs; i++) { | |
94 | data = irq_get_irq_data(virq + i); | |
95 | cpumask_copy(data->common->affinity, dest); | |
96 | data->common->ipi_offset = offset; | |
4589f450 | 97 | irq_set_status_flags(virq + i, IRQ_NO_BALANCING); |
d17bf24e QY |
98 | } |
99 | return virq; | |
100 | ||
101 | free_descs: | |
102 | irq_free_descs(virq, nr_irqs); | |
103 | return 0; | |
104 | } | |
105 | ||
106 | /** | |
107 | * irq_destroy_ipi() - unreserve an IPI that was previously allocated | |
108 | * @irq: linux irq number to be destroyed | |
01292cea | 109 | * @dest: cpumask of cpus which should have the IPI removed |
d17bf24e QY |
110 | * |
111 | * Return the IPIs allocated with irq_reserve_ipi() to the system destroying | |
112 | * all virqs associated with them. | |
113 | */ | |
01292cea | 114 | void irq_destroy_ipi(unsigned int irq, const struct cpumask *dest) |
d17bf24e QY |
115 | { |
116 | struct irq_data *data = irq_get_irq_data(irq); | |
117 | struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; | |
118 | struct irq_domain *domain; | |
119 | unsigned int nr_irqs; | |
120 | ||
121 | if (!irq || !data || !ipimask) | |
122 | return; | |
123 | ||
124 | domain = data->domain; | |
125 | if (WARN_ON(domain == NULL)) | |
126 | return; | |
127 | ||
128 | if (!irq_domain_is_ipi(domain)) { | |
129 | pr_warn("Trying to destroy a non IPI domain!\n"); | |
130 | return; | |
131 | } | |
132 | ||
01292cea MR |
133 | if (WARN_ON(!cpumask_subset(dest, ipimask))) |
134 | /* | |
135 | * Must be destroying a subset of CPUs to which this IPI | |
136 | * was set up to target | |
137 | */ | |
138 | return; | |
139 | ||
140 | if (irq_domain_is_ipi_per_cpu(domain)) { | |
141 | irq = irq + cpumask_first(dest) - data->common->ipi_offset; | |
142 | nr_irqs = cpumask_weight(dest); | |
143 | } else { | |
d17bf24e | 144 | nr_irqs = 1; |
01292cea | 145 | } |
d17bf24e QY |
146 | |
147 | irq_domain_free_irqs(irq, nr_irqs); | |
148 | } | |
f9bce791 QY |
149 | |
150 | /** | |
151 | * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu | |
152 | * @irq: linux irq number | |
153 | * @cpu: the target cpu | |
154 | * | |
155 | * When dealing with coprocessors IPI, we need to inform the coprocessor of | |
156 | * the hwirq it needs to use to receive and send IPIs. | |
157 | * | |
158 | * Returns hwirq value on success and INVALID_HWIRQ on failure. | |
159 | */ | |
160 | irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu) | |
161 | { | |
162 | struct irq_data *data = irq_get_irq_data(irq); | |
163 | struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; | |
164 | ||
165 | if (!data || !ipimask || cpu > nr_cpu_ids) | |
166 | return INVALID_HWIRQ; | |
167 | ||
168 | if (!cpumask_test_cpu(cpu, ipimask)) | |
169 | return INVALID_HWIRQ; | |
170 | ||
171 | /* | |
172 | * Get the real hardware irq number if the underlying implementation | |
173 | * uses a seperate irq per cpu. If the underlying implementation uses | |
174 | * a single hardware irq for all cpus then the IPI send mechanism | |
3b8e29a8 | 175 | * needs to take care of the cpu destinations. |
f9bce791 QY |
176 | */ |
177 | if (irq_domain_is_ipi_per_cpu(data->domain)) | |
178 | data = irq_get_irq_data(irq + cpu - data->common->ipi_offset); | |
179 | ||
180 | return data ? irqd_to_hwirq(data) : INVALID_HWIRQ; | |
181 | } | |
182 | EXPORT_SYMBOL_GPL(ipi_get_hwirq); | |
3b8e29a8 QY |
183 | |
184 | static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data, | |
185 | const struct cpumask *dest, unsigned int cpu) | |
186 | { | |
187 | struct cpumask *ipimask = irq_data_get_affinity_mask(data); | |
188 | ||
189 | if (!chip || !ipimask) | |
190 | return -EINVAL; | |
191 | ||
192 | if (!chip->ipi_send_single && !chip->ipi_send_mask) | |
193 | return -EINVAL; | |
194 | ||
195 | if (cpu > nr_cpu_ids) | |
196 | return -EINVAL; | |
197 | ||
198 | if (dest) { | |
199 | if (!cpumask_subset(dest, ipimask)) | |
200 | return -EINVAL; | |
201 | } else { | |
202 | if (!cpumask_test_cpu(cpu, ipimask)) | |
203 | return -EINVAL; | |
204 | } | |
205 | return 0; | |
206 | } | |
207 | ||
208 | /** | |
209 | * __ipi_send_single - send an IPI to a target Linux SMP CPU | |
210 | * @desc: pointer to irq_desc of the IRQ | |
211 | * @cpu: destination CPU, must in the destination mask passed to | |
212 | * irq_reserve_ipi() | |
213 | * | |
214 | * This function is for architecture or core code to speed up IPI sending. Not | |
215 | * usable from driver code. | |
216 | * | |
217 | * Returns zero on success and negative error number on failure. | |
218 | */ | |
219 | int __ipi_send_single(struct irq_desc *desc, unsigned int cpu) | |
220 | { | |
221 | struct irq_data *data = irq_desc_get_irq_data(desc); | |
222 | struct irq_chip *chip = irq_data_get_irq_chip(data); | |
223 | ||
224 | #ifdef DEBUG | |
225 | /* | |
226 | * Minimise the overhead by omitting the checks for Linux SMP IPIs. | |
227 | * Since the callers should be arch or core code which is generally | |
228 | * trusted, only check for errors when debugging. | |
229 | */ | |
230 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) | |
231 | return -EINVAL; | |
232 | #endif | |
233 | if (!chip->ipi_send_single) { | |
234 | chip->ipi_send_mask(data, cpumask_of(cpu)); | |
235 | return 0; | |
236 | } | |
237 | ||
238 | /* FIXME: Store this information in irqdata flags */ | |
239 | if (irq_domain_is_ipi_per_cpu(data->domain) && | |
240 | cpu != data->common->ipi_offset) { | |
241 | /* use the correct data for that cpu */ | |
242 | unsigned irq = data->irq + cpu - data->common->ipi_offset; | |
243 | ||
244 | data = irq_get_irq_data(irq); | |
245 | } | |
246 | chip->ipi_send_single(data, cpu); | |
247 | return 0; | |
248 | } | |
249 | ||
250 | /** | |
251 | * ipi_send_mask - send an IPI to target Linux SMP CPU(s) | |
252 | * @desc: pointer to irq_desc of the IRQ | |
253 | * @dest: dest CPU(s), must be a subset of the mask passed to | |
254 | * irq_reserve_ipi() | |
255 | * | |
256 | * This function is for architecture or core code to speed up IPI sending. Not | |
257 | * usable from driver code. | |
258 | * | |
259 | * Returns zero on success and negative error number on failure. | |
260 | */ | |
261 | int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest) | |
262 | { | |
263 | struct irq_data *data = irq_desc_get_irq_data(desc); | |
264 | struct irq_chip *chip = irq_data_get_irq_chip(data); | |
265 | unsigned int cpu; | |
266 | ||
267 | #ifdef DEBUG | |
268 | /* | |
269 | * Minimise the overhead by omitting the checks for Linux SMP IPIs. | |
270 | * Since the callers should be arch or core code which is generally | |
271 | * trusted, only check for errors when debugging. | |
272 | */ | |
273 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) | |
274 | return -EINVAL; | |
275 | #endif | |
276 | if (chip->ipi_send_mask) { | |
277 | chip->ipi_send_mask(data, dest); | |
278 | return 0; | |
279 | } | |
280 | ||
281 | if (irq_domain_is_ipi_per_cpu(data->domain)) { | |
282 | unsigned int base = data->irq; | |
283 | ||
284 | for_each_cpu(cpu, dest) { | |
285 | unsigned irq = base + cpu - data->common->ipi_offset; | |
286 | ||
287 | data = irq_get_irq_data(irq); | |
288 | chip->ipi_send_single(data, cpu); | |
289 | } | |
290 | } else { | |
291 | for_each_cpu(cpu, dest) | |
292 | chip->ipi_send_single(data, cpu); | |
293 | } | |
294 | return 0; | |
295 | } | |
296 | ||
297 | /** | |
298 | * ipi_send_single - Send an IPI to a single CPU | |
299 | * @virq: linux irq number from irq_reserve_ipi() | |
300 | * @cpu: destination CPU, must in the destination mask passed to | |
301 | * irq_reserve_ipi() | |
302 | * | |
303 | * Returns zero on success and negative error number on failure. | |
304 | */ | |
305 | int ipi_send_single(unsigned int virq, unsigned int cpu) | |
306 | { | |
307 | struct irq_desc *desc = irq_to_desc(virq); | |
308 | struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; | |
309 | struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; | |
310 | ||
311 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) | |
312 | return -EINVAL; | |
313 | ||
314 | return __ipi_send_single(desc, cpu); | |
315 | } | |
316 | EXPORT_SYMBOL_GPL(ipi_send_single); | |
317 | ||
318 | /** | |
319 | * ipi_send_mask - Send an IPI to target CPU(s) | |
320 | * @virq: linux irq number from irq_reserve_ipi() | |
321 | * @dest: dest CPU(s), must be a subset of the mask passed to | |
322 | * irq_reserve_ipi() | |
323 | * | |
324 | * Returns zero on success and negative error number on failure. | |
325 | */ | |
326 | int ipi_send_mask(unsigned int virq, const struct cpumask *dest) | |
327 | { | |
328 | struct irq_desc *desc = irq_to_desc(virq); | |
329 | struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; | |
330 | struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; | |
331 | ||
332 | if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) | |
333 | return -EINVAL; | |
334 | ||
335 | return __ipi_send_mask(desc, dest); | |
336 | } | |
337 | EXPORT_SYMBOL_GPL(ipi_send_mask); |