Commit | Line | Data |
---|---|---|
44380982 JL |
1 | /* |
2 | * Support of MSI, HPET and DMAR interrupts. | |
3 | * | |
4 | * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo | |
5 | * Moved from arch/x86/kernel/apic/io_apic.c. | |
52f518a3 JL |
6 | * Jiang Liu <jiang.liu@linux.intel.com> |
7 | * Convert to hierarchical irqdomain | |
44380982 JL |
8 | * |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License version 2 as | |
11 | * published by the Free Software Foundation. | |
12 | */ | |
13 | #include <linux/mm.h> | |
14 | #include <linux/interrupt.h> | |
15 | #include <linux/pci.h> | |
16 | #include <linux/dmar.h> | |
17 | #include <linux/hpet.h> | |
18 | #include <linux/msi.h> | |
4c8f9960 | 19 | #include <linux/irqdomain.h> |
44380982 JL |
20 | #include <asm/msidef.h> |
21 | #include <asm/hpet.h> | |
22 | #include <asm/hw_irq.h> | |
23 | #include <asm/apic.h> | |
24 | #include <asm/irq_remapping.h> | |
25 | ||
52f518a3 JL |
26 | static struct irq_domain *msi_default_domain; |
27 | ||
44380982 JL |
28 | void native_compose_msi_msg(struct pci_dev *pdev, |
29 | unsigned int irq, unsigned int dest, | |
30 | struct msi_msg *msg, u8 hpet_id) | |
31 | { | |
32 | struct irq_cfg *cfg = irq_cfg(irq); | |
33 | ||
34 | msg->address_hi = MSI_ADDR_BASE_HI; | |
35 | ||
36 | if (x2apic_enabled()) | |
37 | msg->address_hi |= MSI_ADDR_EXT_DEST_ID(dest); | |
38 | ||
39 | msg->address_lo = | |
40 | MSI_ADDR_BASE_LO | | |
41 | ((apic->irq_dest_mode == 0) ? | |
42 | MSI_ADDR_DEST_MODE_PHYSICAL : | |
43 | MSI_ADDR_DEST_MODE_LOGICAL) | | |
44 | ((apic->irq_delivery_mode != dest_LowestPrio) ? | |
45 | MSI_ADDR_REDIRECTION_CPU : | |
46 | MSI_ADDR_REDIRECTION_LOWPRI) | | |
47 | MSI_ADDR_DEST_ID(dest); | |
48 | ||
49 | msg->data = | |
50 | MSI_DATA_TRIGGER_EDGE | | |
51 | MSI_DATA_LEVEL_ASSERT | | |
52 | ((apic->irq_delivery_mode != dest_LowestPrio) ? | |
53 | MSI_DATA_DELIVERY_FIXED : | |
54 | MSI_DATA_DELIVERY_LOWPRI) | | |
55 | MSI_DATA_VECTOR(cfg->vector); | |
56 | } | |
57 | ||
3cb96f0c JL |
58 | static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) |
59 | { | |
60 | struct irq_cfg *cfg = irqd_cfg(data); | |
61 | ||
62 | msg->address_hi = MSI_ADDR_BASE_HI; | |
63 | ||
64 | if (x2apic_enabled()) | |
65 | msg->address_hi |= MSI_ADDR_EXT_DEST_ID(cfg->dest_apicid); | |
66 | ||
67 | msg->address_lo = | |
68 | MSI_ADDR_BASE_LO | | |
69 | ((apic->irq_dest_mode == 0) ? | |
70 | MSI_ADDR_DEST_MODE_PHYSICAL : | |
71 | MSI_ADDR_DEST_MODE_LOGICAL) | | |
72 | ((apic->irq_delivery_mode != dest_LowestPrio) ? | |
73 | MSI_ADDR_REDIRECTION_CPU : | |
74 | MSI_ADDR_REDIRECTION_LOWPRI) | | |
75 | MSI_ADDR_DEST_ID(cfg->dest_apicid); | |
76 | ||
77 | msg->data = | |
78 | MSI_DATA_TRIGGER_EDGE | | |
79 | MSI_DATA_LEVEL_ASSERT | | |
80 | ((apic->irq_delivery_mode != dest_LowestPrio) ? | |
81 | MSI_DATA_DELIVERY_FIXED : | |
82 | MSI_DATA_DELIVERY_LOWPRI) | | |
83 | MSI_DATA_VECTOR(cfg->vector); | |
84 | } | |
85 | ||
86 | static void msi_update_msg(struct msi_msg *msg, struct irq_data *irq_data) | |
87 | { | |
88 | struct irq_cfg *cfg = irqd_cfg(irq_data); | |
89 | ||
90 | msg->data &= ~MSI_DATA_VECTOR_MASK; | |
91 | msg->data |= MSI_DATA_VECTOR(cfg->vector); | |
92 | msg->address_lo &= ~MSI_ADDR_DEST_ID_MASK; | |
93 | msg->address_lo |= MSI_ADDR_DEST_ID(cfg->dest_apicid); | |
94 | } | |
95 | ||
44380982 JL |
96 | static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, |
97 | struct msi_msg *msg, u8 hpet_id) | |
98 | { | |
99 | struct irq_cfg *cfg; | |
100 | int err; | |
101 | unsigned dest; | |
102 | ||
103 | if (disable_apic) | |
104 | return -ENXIO; | |
105 | ||
106 | cfg = irq_cfg(irq); | |
107 | err = assign_irq_vector(irq, cfg, apic->target_cpus()); | |
108 | if (err) | |
109 | return err; | |
110 | ||
111 | err = apic->cpu_mask_to_apicid_and(cfg->domain, | |
112 | apic->target_cpus(), &dest); | |
113 | if (err) | |
114 | return err; | |
115 | ||
116 | x86_msi.compose_msi_msg(pdev, irq, dest, msg, hpet_id); | |
117 | ||
118 | return 0; | |
119 | } | |
120 | ||
44380982 JL |
121 | /* |
122 | * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices, | |
123 | * which implement the MSI or MSI-X Capability Structure. | |
124 | */ | |
52f518a3 | 125 | static struct irq_chip pci_msi_controller = { |
44380982 JL |
126 | .name = "PCI-MSI", |
127 | .irq_unmask = pci_msi_unmask_irq, | |
128 | .irq_mask = pci_msi_mask_irq, | |
52f518a3 JL |
129 | .irq_ack = irq_chip_ack_parent, |
130 | .irq_set_affinity = msi_domain_set_affinity, | |
131 | .irq_retrigger = irq_chip_retrigger_hierarchy, | |
132 | .irq_print_chip = irq_remapping_print_chip, | |
133 | .irq_compose_msi_msg = irq_msi_compose_msg, | |
134 | .irq_write_msi_msg = pci_msi_domain_write_msg, | |
44380982 JL |
135 | .flags = IRQCHIP_SKIP_SET_WAKE, |
136 | }; | |
137 | ||
52f518a3 | 138 | int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) |
44380982 | 139 | { |
52f518a3 JL |
140 | struct irq_domain *domain; |
141 | struct irq_alloc_info info; | |
44380982 | 142 | |
52f518a3 JL |
143 | init_irq_alloc_info(&info, NULL); |
144 | info.type = X86_IRQ_ALLOC_TYPE_MSI; | |
145 | info.msi_dev = dev; | |
44380982 | 146 | |
52f518a3 JL |
147 | domain = irq_remapping_get_irq_domain(&info); |
148 | if (domain == NULL) | |
149 | domain = msi_default_domain; | |
150 | if (domain == NULL) | |
151 | return -ENOSYS; | |
44380982 | 152 | |
52f518a3 JL |
153 | return pci_msi_domain_alloc_irqs(domain, dev, nvec, type); |
154 | } | |
44380982 | 155 | |
52f518a3 JL |
156 | void native_teardown_msi_irq(unsigned int irq) |
157 | { | |
158 | irq_domain_free_irqs(irq, 1); | |
159 | } | |
44380982 | 160 | |
52f518a3 JL |
161 | static irq_hw_number_t pci_msi_get_hwirq(struct msi_domain_info *info, |
162 | msi_alloc_info_t *arg) | |
163 | { | |
164 | return arg->msi_hwirq; | |
165 | } | |
44380982 | 166 | |
52f518a3 JL |
167 | static int pci_msi_prepare(struct irq_domain *domain, struct device *dev, |
168 | int nvec, msi_alloc_info_t *arg) | |
169 | { | |
170 | struct pci_dev *pdev = to_pci_dev(dev); | |
171 | struct msi_desc *desc = first_pci_msi_entry(pdev); | |
172 | ||
173 | init_irq_alloc_info(arg, NULL); | |
174 | arg->msi_dev = pdev; | |
175 | if (desc->msi_attrib.is_msix) { | |
176 | arg->type = X86_IRQ_ALLOC_TYPE_MSIX; | |
177 | } else { | |
178 | arg->type = X86_IRQ_ALLOC_TYPE_MSI; | |
179 | arg->flags |= X86_IRQ_ALLOC_CONTIGUOUS_VECTORS; | |
180 | } | |
44380982 JL |
181 | |
182 | return 0; | |
183 | } | |
184 | ||
52f518a3 | 185 | static void pci_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) |
44380982 | 186 | { |
52f518a3 JL |
187 | arg->msi_hwirq = pci_msi_domain_calc_hwirq(arg->msi_dev, desc); |
188 | } | |
44380982 | 189 | |
52f518a3 JL |
190 | static struct msi_domain_ops pci_msi_domain_ops = { |
191 | .get_hwirq = pci_msi_get_hwirq, | |
192 | .msi_prepare = pci_msi_prepare, | |
193 | .set_desc = pci_msi_set_desc, | |
194 | }; | |
44380982 | 195 | |
52f518a3 JL |
196 | static struct msi_domain_info pci_msi_domain_info = { |
197 | .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | | |
198 | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, | |
199 | .ops = &pci_msi_domain_ops, | |
200 | .chip = &pci_msi_controller, | |
201 | .handler = handle_edge_irq, | |
202 | .handler_name = "edge", | |
203 | }; | |
44380982 | 204 | |
52f518a3 JL |
205 | void arch_init_msi_domain(struct irq_domain *parent) |
206 | { | |
207 | if (disable_apic) | |
208 | return; | |
44380982 | 209 | |
52f518a3 JL |
210 | msi_default_domain = pci_msi_create_irq_domain(NULL, |
211 | &pci_msi_domain_info, parent); | |
212 | if (!msi_default_domain) | |
213 | pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); | |
44380982 JL |
214 | } |
215 | ||
52f518a3 JL |
216 | #ifdef CONFIG_IRQ_REMAP |
217 | struct irq_domain *arch_create_msi_irq_domain(struct irq_domain *parent) | |
44380982 | 218 | { |
52f518a3 | 219 | return msi_create_irq_domain(NULL, &pci_msi_domain_info, parent); |
44380982 | 220 | } |
52f518a3 | 221 | #endif |
44380982 JL |
222 | |
223 | #ifdef CONFIG_DMAR_TABLE | |
224 | static int | |
225 | dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask, | |
226 | bool force) | |
227 | { | |
a9786091 | 228 | struct irq_cfg *cfg = irqd_cfg(data); |
44380982 JL |
229 | unsigned int dest, irq = data->irq; |
230 | struct msi_msg msg; | |
231 | int ret; | |
232 | ||
233 | ret = apic_set_affinity(data, mask, &dest); | |
234 | if (ret) | |
235 | return ret; | |
236 | ||
237 | dmar_msi_read(irq, &msg); | |
238 | ||
239 | msg.data &= ~MSI_DATA_VECTOR_MASK; | |
240 | msg.data |= MSI_DATA_VECTOR(cfg->vector); | |
241 | msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK; | |
242 | msg.address_lo |= MSI_ADDR_DEST_ID(dest); | |
243 | msg.address_hi = MSI_ADDR_BASE_HI | MSI_ADDR_EXT_DEST_ID(dest); | |
244 | ||
245 | dmar_msi_write(irq, &msg); | |
246 | ||
247 | return IRQ_SET_MASK_OK_NOCOPY; | |
248 | } | |
249 | ||
250 | static struct irq_chip dmar_msi_type = { | |
251 | .name = "DMAR_MSI", | |
252 | .irq_unmask = dmar_msi_unmask, | |
253 | .irq_mask = dmar_msi_mask, | |
254 | .irq_ack = apic_ack_edge, | |
255 | .irq_set_affinity = dmar_msi_set_affinity, | |
256 | .irq_retrigger = apic_retrigger_irq, | |
257 | .flags = IRQCHIP_SKIP_SET_WAKE, | |
258 | }; | |
259 | ||
260 | int arch_setup_dmar_msi(unsigned int irq) | |
261 | { | |
262 | int ret; | |
263 | struct msi_msg msg; | |
264 | ||
265 | ret = msi_compose_msg(NULL, irq, &msg, -1); | |
266 | if (ret < 0) | |
267 | return ret; | |
268 | dmar_msi_write(irq, &msg); | |
269 | irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq, | |
270 | "edge"); | |
271 | return 0; | |
272 | } | |
a62b32cd JL |
273 | |
274 | int dmar_alloc_hwirq(void) | |
275 | { | |
276 | return irq_domain_alloc_irqs(NULL, 1, NUMA_NO_NODE, NULL); | |
277 | } | |
278 | ||
279 | void dmar_free_hwirq(int irq) | |
280 | { | |
281 | irq_domain_free_irqs(irq, 1); | |
282 | } | |
44380982 JL |
283 | #endif |
284 | ||
285 | /* | |
286 | * MSI message composition | |
287 | */ | |
288 | #ifdef CONFIG_HPET_TIMER | |
3cb96f0c JL |
289 | static inline int hpet_dev_id(struct irq_domain *domain) |
290 | { | |
291 | return (int)(long)domain->host_data; | |
292 | } | |
44380982 JL |
293 | |
294 | static int hpet_msi_set_affinity(struct irq_data *data, | |
295 | const struct cpumask *mask, bool force) | |
296 | { | |
3cb96f0c | 297 | struct irq_data *parent = data->parent_data; |
44380982 | 298 | struct msi_msg msg; |
44380982 JL |
299 | int ret; |
300 | ||
3cb96f0c JL |
301 | ret = parent->chip->irq_set_affinity(parent, mask, force); |
302 | if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) { | |
303 | hpet_msi_read(data->handler_data, &msg); | |
304 | msi_update_msg(&msg, data); | |
305 | hpet_msi_write(data->handler_data, &msg); | |
306 | } | |
44380982 | 307 | |
3cb96f0c | 308 | return ret; |
44380982 JL |
309 | } |
310 | ||
3cb96f0c | 311 | static struct irq_chip hpet_msi_controller = { |
44380982 JL |
312 | .name = "HPET_MSI", |
313 | .irq_unmask = hpet_msi_unmask, | |
314 | .irq_mask = hpet_msi_mask, | |
3cb96f0c | 315 | .irq_ack = irq_chip_ack_parent, |
44380982 | 316 | .irq_set_affinity = hpet_msi_set_affinity, |
3cb96f0c JL |
317 | .irq_retrigger = irq_chip_retrigger_hierarchy, |
318 | .irq_print_chip = irq_remapping_print_chip, | |
319 | .irq_compose_msi_msg = irq_msi_compose_msg, | |
44380982 JL |
320 | .flags = IRQCHIP_SKIP_SET_WAKE, |
321 | }; | |
322 | ||
323 | int default_setup_hpet_msi(unsigned int irq, unsigned int id) | |
324 | { | |
3cb96f0c | 325 | struct irq_chip *chip = &hpet_msi_controller; |
44380982 JL |
326 | struct msi_msg msg; |
327 | int ret; | |
328 | ||
329 | ret = msi_compose_msg(NULL, irq, &msg, id); | |
330 | if (ret < 0) | |
331 | return ret; | |
332 | ||
333 | hpet_msi_write(irq_get_handler_data(irq), &msg); | |
334 | irq_set_status_flags(irq, IRQ_MOVE_PCNTXT); | |
335 | setup_remapped_irq(irq, irq_cfg(irq), chip); | |
336 | ||
337 | irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge"); | |
338 | return 0; | |
339 | } | |
3cb96f0c JL |
340 | |
341 | static int hpet_domain_alloc(struct irq_domain *domain, unsigned int virq, | |
342 | unsigned int nr_irqs, void *arg) | |
343 | { | |
344 | struct irq_alloc_info *info = arg; | |
345 | int ret; | |
346 | ||
347 | if (nr_irqs > 1 || !info || info->type != X86_IRQ_ALLOC_TYPE_HPET) | |
348 | return -EINVAL; | |
349 | if (irq_find_mapping(domain, info->hpet_index)) { | |
350 | pr_warn("IRQ for HPET%d already exists.\n", info->hpet_index); | |
351 | return -EEXIST; | |
352 | } | |
353 | ||
354 | ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); | |
355 | if (ret >= 0) { | |
356 | irq_set_status_flags(virq, IRQ_MOVE_PCNTXT); | |
357 | irq_domain_set_hwirq_and_chip(domain, virq, info->hpet_index, | |
358 | &hpet_msi_controller, NULL); | |
359 | irq_set_handler_data(virq, info->hpet_data); | |
360 | __irq_set_handler(virq, handle_edge_irq, 0, "edge"); | |
361 | } | |
362 | ||
363 | return ret; | |
364 | } | |
365 | ||
366 | static void hpet_domain_free(struct irq_domain *domain, unsigned int virq, | |
367 | unsigned int nr_irqs) | |
368 | { | |
369 | BUG_ON(nr_irqs > 1); | |
370 | irq_clear_status_flags(virq, IRQ_MOVE_PCNTXT); | |
371 | irq_domain_free_irqs_top(domain, virq, nr_irqs); | |
372 | } | |
373 | ||
374 | static void hpet_domain_activate(struct irq_domain *domain, | |
375 | struct irq_data *irq_data) | |
376 | { | |
377 | struct msi_msg msg; | |
378 | ||
379 | BUG_ON(irq_chip_compose_msi_msg(irq_data, &msg)); | |
380 | hpet_msi_write(irq_get_handler_data(irq_data->irq), &msg); | |
381 | } | |
382 | ||
383 | static void hpet_domain_deactivate(struct irq_domain *domain, | |
384 | struct irq_data *irq_data) | |
385 | { | |
386 | struct msi_msg msg; | |
387 | ||
388 | memset(&msg, 0, sizeof(msg)); | |
389 | hpet_msi_write(irq_get_handler_data(irq_data->irq), &msg); | |
390 | } | |
391 | ||
392 | static struct irq_domain_ops hpet_domain_ops = { | |
393 | .alloc = hpet_domain_alloc, | |
394 | .free = hpet_domain_free, | |
395 | .activate = hpet_domain_activate, | |
396 | .deactivate = hpet_domain_deactivate, | |
397 | }; | |
398 | ||
399 | struct irq_domain *hpet_create_irq_domain(int hpet_id) | |
400 | { | |
401 | struct irq_domain *parent; | |
402 | struct irq_alloc_info info; | |
403 | ||
404 | if (x86_vector_domain == NULL) | |
405 | return NULL; | |
406 | ||
407 | init_irq_alloc_info(&info, NULL); | |
408 | info.type = X86_IRQ_ALLOC_TYPE_HPET; | |
409 | info.hpet_id = hpet_id; | |
410 | parent = irq_remapping_get_ir_irq_domain(&info); | |
411 | if (parent == NULL) | |
412 | parent = x86_vector_domain; | |
413 | ||
414 | return irq_domain_add_hierarchy(parent, 0, 0, NULL, &hpet_domain_ops, | |
415 | (void *)(long)hpet_id); | |
416 | } | |
417 | ||
418 | int hpet_assign_irq(struct irq_domain *domain, struct hpet_dev *dev, | |
419 | int dev_num) | |
420 | { | |
421 | struct irq_alloc_info info; | |
422 | ||
423 | init_irq_alloc_info(&info, NULL); | |
424 | info.type = X86_IRQ_ALLOC_TYPE_HPET; | |
425 | info.hpet_data = dev; | |
426 | info.hpet_id = hpet_dev_id(domain); | |
427 | info.hpet_index = dev_num; | |
428 | ||
429 | return irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, NULL); | |
430 | } | |
44380982 | 431 | #endif |