projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc/powernv/ioda2: Introduce pnv_pci_ioda2_set_window
[deliverable/linux.git] / arch / powerpc / platforms / powernv / pci-ioda.c
CommitLineData
184cd4a3
BH		 1/*
2 * Support PCI/PCIe on PowerNV platforms
3 *
4 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
cee72d5b 12#undef DEBUG
184cd4a3
BH		 13
14#include <linux/kernel.h>
15#include <linux/pci.h>
361f2a2a 16#include <linux/crash_dump.h>
37c367f2 17#include <linux/debugfs.h>
184cd4a3
BH		 18#include <linux/delay.h>
19#include <linux/string.h>
20#include <linux/init.h>
21#include <linux/bootmem.h>
22#include <linux/irq.h>
23#include <linux/io.h>
24#include <linux/msi.h>
cd15b048 25#include <linux/memblock.h>
ac9a5889 26#include <linux/iommu.h>
e57080f1 27#include <linux/rculist.h>
184cd4a3
BH		 28
29#include <asm/sections.h>
30#include <asm/io.h>
31#include <asm/prom.h>
32#include <asm/pci-bridge.h>
33#include <asm/machdep.h>
fb1b55d6 34#include <asm/msi_bitmap.h>
184cd4a3
BH		 35#include <asm/ppc-pci.h>
36#include <asm/opal.h>
37#include <asm/iommu.h>
38#include <asm/tce.h>
137436c9 39#include <asm/xics.h>
37c367f2 40#include <asm/debug.h>
262af557 41#include <asm/firmware.h>
80c49c7e 42#include <asm/pnv-pci.h>
aca6913f 43#include <asm/mmzone.h>
80c49c7e 44
ec249dd8 45#include <misc/cxl-base.h>
184cd4a3
BH		 46
47#include "powernv.h"
48#include "pci.h"
49
781a868f
WY		 50/* 256M DMA window, 4K TCE pages, 8 bytes TCE */
51#define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8)
52
aca6913f
AK		 53static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
54
6d31c2fa
JP		 55static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
56 const char *fmt, ...)
57{
58 struct va_format vaf;
59 va_list args;
60 char pfix[32];
61
62 va_start(args, fmt);
63
64 vaf.fmt = fmt;
65 vaf.va = &args;
66
781a868f 67 if (pe->flags & PNV_IODA_PE_DEV)
6d31c2fa 68 strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
781a868f 69 else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
6d31c2fa
JP		 70 sprintf(pfix, "%04x:%02x ",
71 pci_domain_nr(pe->pbus), pe->pbus->number);
781a868f
WY		 72#ifdef CONFIG_PCI_IOV
73 else if (pe->flags & PNV_IODA_PE_VF)
74 sprintf(pfix, "%04x:%02x:%2x.%d",
75 pci_domain_nr(pe->parent_dev->bus),
76 (pe->rid & 0xff00) >> 8,
77 PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
78#endif /* CONFIG_PCI_IOV*/
6d31c2fa
JP		 79
80 printk("%spci %s: [PE# %.3d] %pV",
81 level, pfix, pe->pe_number, &vaf);
82
83 va_end(args);
84}
184cd4a3 85
6d31c2fa
JP		 86#define pe_err(pe, fmt, ...) \
87 pe_level_printk(pe, KERN_ERR, fmt, ##__VA_ARGS__)
88#define pe_warn(pe, fmt, ...) \
89 pe_level_printk(pe, KERN_WARNING, fmt, ##__VA_ARGS__)
90#define pe_info(pe, fmt, ...) \
91 pe_level_printk(pe, KERN_INFO, fmt, ##__VA_ARGS__)
184cd4a3 92
4e287840
TLSC		 93static bool pnv_iommu_bypass_disabled __read_mostly;
94
95static int __init iommu_setup(char *str)
96{
97 if (!str)
98 return -EINVAL;
99
100 while (*str) {
101 if (!strncmp(str, "nobypass", 8)) {
102 pnv_iommu_bypass_disabled = true;
103 pr_info("PowerNV: IOMMU bypass window disabled.\n");
104 break;
105 }
106 str += strcspn(str, ",");
107 if (*str == ',')
108 str++;
109 }
110
111 return 0;
112}
113early_param("iommu", iommu_setup);
114
8e0a1611
AK		 115/*
116 * stdcix is only supposed to be used in hypervisor real mode as per
117 * the architecture spec
118 */
119static inline void __raw_rm_writeq(u64 val, volatile void __iomem *paddr)
120{
121 __asm__ __volatile__("stdcix %0,0,%1"
122 : : "r" (val), "r" (paddr) : "memory");
123}
124
262af557
GC		 125static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
126{
127 return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
128 (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
129}
130
4b82ab18
GS		 131static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
132{
133 if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe)) {
134 pr_warn("%s: Invalid PE %d on PHB#%x\n",
135 __func__, pe_no, phb->hose->global_number);
136 return;
137 }
138
139 if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) {
140 pr_warn("%s: PE %d was assigned on PHB#%x\n",
141 __func__, pe_no, phb->hose->global_number);
142 return;
143 }
144
145 phb->ioda.pe_array[pe_no].phb = phb;
146 phb->ioda.pe_array[pe_no].pe_number = pe_no;
147}
148
cad5cef6 149static int pnv_ioda_alloc_pe(struct pnv_phb *phb)
184cd4a3
BH		 150{
151 unsigned long pe;
152
153 do {
154 pe = find_next_zero_bit(phb->ioda.pe_alloc,
155 phb->ioda.total_pe, 0);
156 if (pe >= phb->ioda.total_pe)
157 return IODA_INVALID_PE;
158 } while(test_and_set_bit(pe, phb->ioda.pe_alloc));
159
4cce9550 160 phb->ioda.pe_array[pe].phb = phb;
184cd4a3
BH		 161 phb->ioda.pe_array[pe].pe_number = pe;
162 return pe;
163}
164
cad5cef6 165static void pnv_ioda_free_pe(struct pnv_phb *phb, int pe)
184cd4a3
BH		 166{
167 WARN_ON(phb->ioda.pe_array[pe].pdev);
168
169 memset(&phb->ioda.pe_array[pe], 0, sizeof(struct pnv_ioda_pe));
170 clear_bit(pe, phb->ioda.pe_alloc);
171}
172
262af557
GC		 173/* The default M64 BAR is shared by all PEs */
174static int pnv_ioda2_init_m64(struct pnv_phb *phb)
175{
176 const char *desc;
177 struct resource *r;
178 s64 rc;
179
180 /* Configure the default M64 BAR */
181 rc = opal_pci_set_phb_mem_window(phb->opal_id,
182 OPAL_M64_WINDOW_TYPE,
183 phb->ioda.m64_bar_idx,
184 phb->ioda.m64_base,
185 0, /* unused */
186 phb->ioda.m64_size);
187 if (rc != OPAL_SUCCESS) {
188 desc = "configuring";
189 goto fail;
190 }
191
192 /* Enable the default M64 BAR */
193 rc = opal_pci_phb_mmio_enable(phb->opal_id,
194 OPAL_M64_WINDOW_TYPE,
195 phb->ioda.m64_bar_idx,
196 OPAL_ENABLE_M64_SPLIT);
197 if (rc != OPAL_SUCCESS) {
198 desc = "enabling";
199 goto fail;
200 }
201
202 /* Mark the M64 BAR assigned */
203 set_bit(phb->ioda.m64_bar_idx, &phb->ioda.m64_bar_alloc);
204
205 /*
206 * Strip off the segment used by the reserved PE, which is
207 * expected to be 0 or last one of PE capabicity.
208 */
209 r = &phb->hose->mem_resources[1];
210 if (phb->ioda.reserved_pe == 0)
211 r->start += phb->ioda.m64_segsize;
212 else if (phb->ioda.reserved_pe == (phb->ioda.total_pe - 1))
213 r->end -= phb->ioda.m64_segsize;
214 else
215 pr_warn(" Cannot strip M64 segment for reserved PE#%d\n",
216 phb->ioda.reserved_pe);
217
218 return 0;
219
220fail:
221 pr_warn(" Failure %lld %s M64 BAR#%d\n",
222 rc, desc, phb->ioda.m64_bar_idx);
223 opal_pci_phb_mmio_enable(phb->opal_id,
224 OPAL_M64_WINDOW_TYPE,
225 phb->ioda.m64_bar_idx,
226 OPAL_DISABLE_M64);
227 return -EIO;
228}
229
5ef73567 230static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb)
262af557
GC		 231{
232 resource_size_t sgsz = phb->ioda.m64_segsize;
233 struct pci_dev *pdev;
234 struct resource *r;
235 int base, step, i;
236
237 /*
238 * Root bus always has full M64 range and root port has
239 * M64 range used in reality. So we're checking root port
240 * instead of root bus.
241 */
242 list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) {
4b82ab18
GS		 243 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
244 r = &pdev->resource[PCI_BRIDGE_RESOURCES + i];
262af557
GC		 245 if (!r->parent ||
246 !pnv_pci_is_mem_pref_64(r->flags))
247 continue;
248
249 base = (r->start - phb->ioda.m64_base) / sgsz;
250 for (step = 0; step < resource_size(r) / sgsz; step++)
4b82ab18 251 pnv_ioda_reserve_pe(phb, base + step);
262af557
GC		 252 }
253 }
254}
255
256static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb,
257 struct pci_bus *bus, int all)
258{
259 resource_size_t segsz = phb->ioda.m64_segsize;
260 struct pci_dev *pdev;
261 struct resource *r;
262 struct pnv_ioda_pe *master_pe, *pe;
263 unsigned long size, *pe_alloc;
264 bool found;
265 int start, i, j;
266
267 /* Root bus shouldn't use M64 */
268 if (pci_is_root_bus(bus))
269 return IODA_INVALID_PE;
270
271 /* We support only one M64 window on each bus */
272 found = false;
273 pci_bus_for_each_resource(bus, r, i) {
274 if (r && r->parent &&
275 pnv_pci_is_mem_pref_64(r->flags)) {
276 found = true;
277 break;
278 }
279 }
280
281 /* No M64 window found ? */
282 if (!found)
283 return IODA_INVALID_PE;
284
285 /* Allocate bitmap */
286 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
287 pe_alloc = kzalloc(size, GFP_KERNEL);
288 if (!pe_alloc) {
289 pr_warn("%s: Out of memory !\n",
290 __func__);
291 return IODA_INVALID_PE;
292 }
293
294 /*
295 * Figure out reserved PE numbers by the PE
296 * the its child PEs.
297 */
298 start = (r->start - phb->ioda.m64_base) / segsz;
299 for (i = 0; i < resource_size(r) / segsz; i++)
300 set_bit(start + i, pe_alloc);
301
302 if (all)
303 goto done;
304
305 /*
306 * If the PE doesn't cover all subordinate buses,
307 * we need subtract from reserved PEs for children.
308 */
309 list_for_each_entry(pdev, &bus->devices, bus_list) {
310 if (!pdev->subordinate)
311 continue;
312
313 pci_bus_for_each_resource(pdev->subordinate, r, i) {
314 if (!r || !r->parent ||
315 !pnv_pci_is_mem_pref_64(r->flags))
316 continue;
317
318 start = (r->start - phb->ioda.m64_base) / segsz;
319 for (j = 0; j < resource_size(r) / segsz ; j++)
320 clear_bit(start + j, pe_alloc);
321 }
322 }
323
324 /*
325 * the current bus might not own M64 window and that's all
326 * contributed by its child buses. For the case, we needn't
327 * pick M64 dependent PE#.
328 */
329 if (bitmap_empty(pe_alloc, phb->ioda.total_pe)) {
330 kfree(pe_alloc);
331 return IODA_INVALID_PE;
332 }
333
334 /*
335 * Figure out the master PE and put all slave PEs to master
336 * PE's list to form compound PE.
337 */
338done:
339 master_pe = NULL;
340 i = -1;
341 while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) <
342 phb->ioda.total_pe) {
343 pe = &phb->ioda.pe_array[i];
262af557
GC		 344
345 if (!master_pe) {
346 pe->flags |= PNV_IODA_PE_MASTER;
347 INIT_LIST_HEAD(&pe->slaves);
348 master_pe = pe;
349 } else {
350 pe->flags |= PNV_IODA_PE_SLAVE;
351 pe->master = master_pe;
352 list_add_tail(&pe->list, &master_pe->slaves);
353 }
354 }
355
356 kfree(pe_alloc);
357 return master_pe->pe_number;
358}
359
360static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
361{
362 struct pci_controller *hose = phb->hose;
363 struct device_node *dn = hose->dn;
364 struct resource *res;
365 const u32 *r;
366 u64 pci_addr;
367
1665c4a8
GS		 368 /* FIXME: Support M64 for P7IOC */
369 if (phb->type != PNV_PHB_IODA2) {
370 pr_info(" Not support M64 window\n");
371 return;
372 }
373
262af557
GC		 374 if (!firmware_has_feature(FW_FEATURE_OPALv3)) {
375 pr_info(" Firmware too old to support M64 window\n");
376 return;
377 }
378
379 r = of_get_property(dn, "ibm,opal-m64-window", NULL);
380 if (!r) {
381 pr_info(" No <ibm,opal-m64-window> on %s\n",
382 dn->full_name);
383 return;
384 }
385
262af557
GC		 386 res = &hose->mem_resources[1];
387 res->start = of_translate_address(dn, r + 2);
388 res->end = res->start + of_read_number(r + 4, 2) - 1;
389 res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
390 pci_addr = of_read_number(r, 2);
391 hose->mem_offset[1] = res->start - pci_addr;
392
393 phb->ioda.m64_size = resource_size(res);
394 phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe;
395 phb->ioda.m64_base = pci_addr;
396
e9863e68
WY		 397 pr_info(" MEM64 0x%016llx..0x%016llx -> 0x%016llx\n",
398 res->start, res->end, pci_addr);
399
262af557
GC		 400 /* Use last M64 BAR to cover M64 window */
401 phb->ioda.m64_bar_idx = 15;
402 phb->init_m64 = pnv_ioda2_init_m64;
5ef73567 403 phb->reserve_m64_pe = pnv_ioda2_reserve_m64_pe;
262af557
GC		 404 phb->pick_m64_pe = pnv_ioda2_pick_m64_pe;
405}
406
49dec922
GS		 407static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
408{
409 struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
410 struct pnv_ioda_pe *slave;
411 s64 rc;
412
413 /* Fetch master PE */
414 if (pe->flags & PNV_IODA_PE_SLAVE) {
415 pe = pe->master;
ec8e4e9d
GS		 416 if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
417 return;
418
49dec922
GS		 419 pe_no = pe->pe_number;
420 }
421
422 /* Freeze master PE */
423 rc = opal_pci_eeh_freeze_set(phb->opal_id,
424 pe_no,
425 OPAL_EEH_ACTION_SET_FREEZE_ALL);
426 if (rc != OPAL_SUCCESS) {
427 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
428 __func__, rc, phb->hose->global_number, pe_no);
429 return;
430 }
431
432 /* Freeze slave PEs */
433 if (!(pe->flags & PNV_IODA_PE_MASTER))
434 return;
435
436 list_for_each_entry(slave, &pe->slaves, list) {
437 rc = opal_pci_eeh_freeze_set(phb->opal_id,
438 slave->pe_number,
439 OPAL_EEH_ACTION_SET_FREEZE_ALL);
440 if (rc != OPAL_SUCCESS)
441 pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
442 __func__, rc, phb->hose->global_number,
443 slave->pe_number);
444 }
445}
446
e51df2c1 447static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
49dec922
GS		 448{
449 struct pnv_ioda_pe *pe, *slave;
450 s64 rc;
451
452 /* Find master PE */
453 pe = &phb->ioda.pe_array[pe_no];
454 if (pe->flags & PNV_IODA_PE_SLAVE) {
455 pe = pe->master;
456 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
457 pe_no = pe->pe_number;
458 }
459
460 /* Clear frozen state for master PE */
461 rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
462 if (rc != OPAL_SUCCESS) {
463 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
464 __func__, rc, opt, phb->hose->global_number, pe_no);
465 return -EIO;
466 }
467
468 if (!(pe->flags & PNV_IODA_PE_MASTER))
469 return 0;
470
471 /* Clear frozen state for slave PEs */
472 list_for_each_entry(slave, &pe->slaves, list) {
473 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
474 slave->pe_number,
475 opt);
476 if (rc != OPAL_SUCCESS) {
477 pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
478 __func__, rc, opt, phb->hose->global_number,
479 slave->pe_number);
480 return -EIO;
481 }
482 }
483
484 return 0;
485}
486
487static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
488{
489 struct pnv_ioda_pe *slave, *pe;
490 u8 fstate, state;
491 __be16 pcierr;
492 s64 rc;
493
494 /* Sanity check on PE number */
495 if (pe_no < 0 || pe_no >= phb->ioda.total_pe)
496 return OPAL_EEH_STOPPED_PERM_UNAVAIL;
497
498 /*
499 * Fetch the master PE and the PE instance might be
500 * not initialized yet.
501 */
502 pe = &phb->ioda.pe_array[pe_no];
503 if (pe->flags & PNV_IODA_PE_SLAVE) {
504 pe = pe->master;
505 WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
506 pe_no = pe->pe_number;
507 }
508
509 /* Check the master PE */
510 rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
511 &state, &pcierr, NULL);
512 if (rc != OPAL_SUCCESS) {
513 pr_warn("%s: Failure %lld getting "
514 "PHB#%x-PE#%x state\n",
515 __func__, rc,
516 phb->hose->global_number, pe_no);
517 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
518 }
519
520 /* Check the slave PE */
521 if (!(pe->flags & PNV_IODA_PE_MASTER))
522 return state;
523
524 list_for_each_entry(slave, &pe->slaves, list) {
525 rc = opal_pci_eeh_freeze_status(phb->opal_id,
526 slave->pe_number,
527 &fstate,
528 &pcierr,
529 NULL);
530 if (rc != OPAL_SUCCESS) {
531 pr_warn("%s: Failure %lld getting "
532 "PHB#%x-PE#%x state\n",
533 __func__, rc,
534 phb->hose->global_number, slave->pe_number);
535 return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
536 }
537
538 /*
539 * Override the result based on the ascending
540 * priority.
541 */
542 if (fstate > state)
543 state = fstate;
544 }
545
546 return state;
547}
548
184cd4a3
BH		 549/* Currently those 2 are only used when MSIs are enabled, this will change
550 * but in the meantime, we need to protect them to avoid warnings
551 */
552#ifdef CONFIG_PCI_MSI
cad5cef6 553static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
184cd4a3
BH		 554{
555 struct pci_controller *hose = pci_bus_to_host(dev->bus);
556 struct pnv_phb *phb = hose->private_data;
b72c1f65 557 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 558
559 if (!pdn)
560 return NULL;
561 if (pdn->pe_number == IODA_INVALID_PE)
562 return NULL;
563 return &phb->ioda.pe_array[pdn->pe_number];
564}
184cd4a3
BH		 565#endif /* CONFIG_PCI_MSI */
566
b131a842
GS		 567static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
568 struct pnv_ioda_pe *parent,
569 struct pnv_ioda_pe *child,
570 bool is_add)
571{
572 const char *desc = is_add ? "adding" : "removing";
573 uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
574 OPAL_REMOVE_PE_FROM_DOMAIN;
575 struct pnv_ioda_pe *slave;
576 long rc;
577
578 /* Parent PE affects child PE */
579 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
580 child->pe_number, op);
581 if (rc != OPAL_SUCCESS) {
582 pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
583 rc, desc);
584 return -ENXIO;
585 }
586
587 if (!(child->flags & PNV_IODA_PE_MASTER))
588 return 0;
589
590 /* Compound case: parent PE affects slave PEs */
591 list_for_each_entry(slave, &child->slaves, list) {
592 rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
593 slave->pe_number, op);
594 if (rc != OPAL_SUCCESS) {
595 pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
596 rc, desc);
597 return -ENXIO;
598 }
599 }
600
601 return 0;
602}
603
604static int pnv_ioda_set_peltv(struct pnv_phb *phb,
605 struct pnv_ioda_pe *pe,
606 bool is_add)
607{
608 struct pnv_ioda_pe *slave;
781a868f 609 struct pci_dev *pdev = NULL;
b131a842
GS		 610 int ret;
611
612 /*
613 * Clear PE frozen state. If it's master PE, we need
614 * clear slave PE frozen state as well.
615 */
616 if (is_add) {
617 opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
618 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
619 if (pe->flags & PNV_IODA_PE_MASTER) {
620 list_for_each_entry(slave, &pe->slaves, list)
621 opal_pci_eeh_freeze_clear(phb->opal_id,
622 slave->pe_number,
623 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
624 }
625 }
626
627 /*
628 * Associate PE in PELT. We need add the PE into the
629 * corresponding PELT-V as well. Otherwise, the error
630 * originated from the PE might contribute to other
631 * PEs.
632 */
633 ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
634 if (ret)
635 return ret;
636
637 /* For compound PEs, any one affects all of them */
638 if (pe->flags & PNV_IODA_PE_MASTER) {
639 list_for_each_entry(slave, &pe->slaves, list) {
640 ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
641 if (ret)
642 return ret;
643 }
644 }
645
646 if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
647 pdev = pe->pbus->self;
781a868f 648 else if (pe->flags & PNV_IODA_PE_DEV)
b131a842 649 pdev = pe->pdev->bus->self;
781a868f
WY		 650#ifdef CONFIG_PCI_IOV
651 else if (pe->flags & PNV_IODA_PE_VF)
652 pdev = pe->parent_dev->bus->self;
653#endif /* CONFIG_PCI_IOV */
b131a842
GS		 654 while (pdev) {
655 struct pci_dn *pdn = pci_get_pdn(pdev);
656 struct pnv_ioda_pe *parent;
657
658 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
659 parent = &phb->ioda.pe_array[pdn->pe_number];
660 ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
661 if (ret)
662 return ret;
663 }
664
665 pdev = pdev->bus->self;
666 }
667
668 return 0;
669}
670
781a868f
WY		 671#ifdef CONFIG_PCI_IOV
672static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
673{
674 struct pci_dev *parent;
675 uint8_t bcomp, dcomp, fcomp;
676 int64_t rc;
677 long rid_end, rid;
678
679 /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
680 if (pe->pbus) {
681 int count;
682
683 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
684 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
685 parent = pe->pbus->self;
686 if (pe->flags & PNV_IODA_PE_BUS_ALL)
687 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
688 else
689 count = 1;
690
691 switch(count) {
692 case 1: bcomp = OpalPciBusAll; break;
693 case 2: bcomp = OpalPciBus7Bits; break;
694 case 4: bcomp = OpalPciBus6Bits; break;
695 case 8: bcomp = OpalPciBus5Bits; break;
696 case 16: bcomp = OpalPciBus4Bits; break;
697 case 32: bcomp = OpalPciBus3Bits; break;
698 default:
699 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
700 count);
701 /* Do an exact match only */
702 bcomp = OpalPciBusAll;
703 }
704 rid_end = pe->rid + (count << 8);
705 } else {
706 if (pe->flags & PNV_IODA_PE_VF)
707 parent = pe->parent_dev;
708 else
709 parent = pe->pdev->bus->self;
710 bcomp = OpalPciBusAll;
711 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
712 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
713 rid_end = pe->rid + 1;
714 }
715
716 /* Clear the reverse map */
717 for (rid = pe->rid; rid < rid_end; rid++)
718 phb->ioda.pe_rmap[rid] = 0;
719
720 /* Release from all parents PELT-V */
721 while (parent) {
722 struct pci_dn *pdn = pci_get_pdn(parent);
723 if (pdn && pdn->pe_number != IODA_INVALID_PE) {
724 rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
725 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
726 /* XXX What to do in case of error ? */
727 }
728 parent = parent->bus->self;
729 }
730
731 opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number,
732 OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
733
734 /* Disassociate PE in PELT */
735 rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
736 pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
737 if (rc)
738 pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
739 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
740 bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
741 if (rc)
742 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
743
744 pe->pbus = NULL;
745 pe->pdev = NULL;
746 pe->parent_dev = NULL;
747
748 return 0;
749}
750#endif /* CONFIG_PCI_IOV */
751
cad5cef6 752static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
184cd4a3
BH		 753{
754 struct pci_dev *parent;
755 uint8_t bcomp, dcomp, fcomp;
756 long rc, rid_end, rid;
757
758 /* Bus validation ? */
759 if (pe->pbus) {
760 int count;
761
762 dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
763 fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
764 parent = pe->pbus->self;
fb446ad0
GS		 765 if (pe->flags & PNV_IODA_PE_BUS_ALL)
766 count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
767 else
768 count = 1;
769
184cd4a3
BH		 770 switch(count) {
771 case 1: bcomp = OpalPciBusAll; break;
772 case 2: bcomp = OpalPciBus7Bits; break;
773 case 4: bcomp = OpalPciBus6Bits; break;
774 case 8: bcomp = OpalPciBus5Bits; break;
775 case 16: bcomp = OpalPciBus4Bits; break;
776 case 32: bcomp = OpalPciBus3Bits; break;
777 default:
781a868f
WY		 778 dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
779 count);
184cd4a3
BH		 780 /* Do an exact match only */
781 bcomp = OpalPciBusAll;
782 }
783 rid_end = pe->rid + (count << 8);
784 } else {
781a868f
WY		 785#ifdef CONFIG_PCI_IOV
786 if (pe->flags & PNV_IODA_PE_VF)
787 parent = pe->parent_dev;
788 else
789#endif /* CONFIG_PCI_IOV */
790 parent = pe->pdev->bus->self;
184cd4a3
BH		 791 bcomp = OpalPciBusAll;
792 dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
793 fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
794 rid_end = pe->rid + 1;
795 }
796
631ad691
GS		 797 /*
798 * Associate PE in PELT. We need add the PE into the
799 * corresponding PELT-V as well. Otherwise, the error
800 * originated from the PE might contribute to other
801 * PEs.
802 */
184cd4a3
BH		 803 rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
804 bcomp, dcomp, fcomp, OPAL_MAP_PE);
805 if (rc) {
806 pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
807 return -ENXIO;
808 }
631ad691 809
b131a842
GS		 810 /* Configure PELTV */
811 pnv_ioda_set_peltv(phb, pe, true);
184cd4a3 812
184cd4a3
BH		 813 /* Setup reverse map */
814 for (rid = pe->rid; rid < rid_end; rid++)
815 phb->ioda.pe_rmap[rid] = pe->pe_number;
816
817 /* Setup one MVTs on IODA1 */
4773f76b
GS		 818 if (phb->type != PNV_PHB_IODA1) {
819 pe->mve_number = 0;
820 goto out;
821 }
822
823 pe->mve_number = pe->pe_number;
824 rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
825 if (rc != OPAL_SUCCESS) {
826 pe_err(pe, "OPAL error %ld setting up MVE %d\n",
827 rc, pe->mve_number);
828 pe->mve_number = -1;
829 } else {
830 rc = opal_pci_set_mve_enable(phb->opal_id,
831 pe->mve_number, OPAL_ENABLE_MVE);
184cd4a3 832 if (rc) {
4773f76b 833 pe_err(pe, "OPAL error %ld enabling MVE %d\n",
184cd4a3
BH		 834 rc, pe->mve_number);
835 pe->mve_number = -1;
184cd4a3 836 }
4773f76b 837 }
184cd4a3 838
4773f76b 839out:
184cd4a3
BH		 840 return 0;
841}
842
cad5cef6
GKH		 843static void pnv_ioda_link_pe_by_weight(struct pnv_phb *phb,
844 struct pnv_ioda_pe *pe)
184cd4a3
BH		 845{
846 struct pnv_ioda_pe *lpe;
847
7ebdf956 848 list_for_each_entry(lpe, &phb->ioda.pe_dma_list, dma_link) {
184cd4a3 849 if (lpe->dma_weight < pe->dma_weight) {
7ebdf956 850 list_add_tail(&pe->dma_link, &lpe->dma_link);
184cd4a3
BH		 851 return;
852 }
853 }
7ebdf956 854 list_add_tail(&pe->dma_link, &phb->ioda.pe_dma_list);
184cd4a3
BH		 855}
856
857static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
858{
859 /* This is quite simplistic. The "base" weight of a device
860 * is 10. 0 means no DMA is to be accounted for it.
861 */
862
863 /* If it's a bridge, no DMA */
864 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
865 return 0;
866
867 /* Reduce the weight of slow USB controllers */
868 if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
869 dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
870 dev->class == PCI_CLASS_SERIAL_USB_EHCI)
871 return 3;
872
873 /* Increase the weight of RAID (includes Obsidian) */
874 if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
875 return 15;
876
877 /* Default */
878 return 10;
879}
880
781a868f
WY		 881#ifdef CONFIG_PCI_IOV
882static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
883{
884 struct pci_dn *pdn = pci_get_pdn(dev);
885 int i;
886 struct resource *res, res2;
887 resource_size_t size;
888 u16 num_vfs;
889
890 if (!dev->is_physfn)
891 return -EINVAL;
892
893 /*
894 * "offset" is in VFs. The M64 windows are sized so that when they
895 * are segmented, each segment is the same size as the IOV BAR.
896 * Each segment is in a separate PE, and the high order bits of the
897 * address are the PE number. Therefore, each VF's BAR is in a
898 * separate PE, and changing the IOV BAR start address changes the
899 * range of PEs the VFs are in.
900 */
901 num_vfs = pdn->num_vfs;
902 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
903 res = &dev->resource[i + PCI_IOV_RESOURCES];
904 if (!res->flags || !res->parent)
905 continue;
906
907 if (!pnv_pci_is_mem_pref_64(res->flags))
908 continue;
909
910 /*
911 * The actual IOV BAR range is determined by the start address
912 * and the actual size for num_vfs VFs BAR. This check is to
913 * make sure that after shifting, the range will not overlap
914 * with another device.
915 */
916 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
917 res2.flags = res->flags;
918 res2.start = res->start + (size * offset);
919 res2.end = res2.start + (size * num_vfs) - 1;
920
921 if (res2.end > res->end) {
922 dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
923 i, &res2, res, num_vfs, offset);
924 return -EBUSY;
925 }
926 }
927
928 /*
929 * After doing so, there would be a "hole" in the /proc/iomem when
930 * offset is a positive value. It looks like the device return some
931 * mmio back to the system, which actually no one could use it.
932 */
933 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
934 res = &dev->resource[i + PCI_IOV_RESOURCES];
935 if (!res->flags || !res->parent)
936 continue;
937
938 if (!pnv_pci_is_mem_pref_64(res->flags))
939 continue;
940
941 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
942 res2 = *res;
943 res->start += size * offset;
944
945 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n",
946 i, &res2, res, num_vfs, offset);
947 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
948 }
949 return 0;
950}
951#endif /* CONFIG_PCI_IOV */
952
fb446ad0 953#if 0
cad5cef6 954static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
184cd4a3
BH		 955{
956 struct pci_controller *hose = pci_bus_to_host(dev->bus);
957 struct pnv_phb *phb = hose->private_data;
b72c1f65 958 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 959 struct pnv_ioda_pe *pe;
960 int pe_num;
961
962 if (!pdn) {
963 pr_err("%s: Device tree node not associated properly\n",
964 pci_name(dev));
965 return NULL;
966 }
967 if (pdn->pe_number != IODA_INVALID_PE)
968 return NULL;
969
970 /* PE#0 has been pre-set */
971 if (dev->bus->number == 0)
972 pe_num = 0;
973 else
974 pe_num = pnv_ioda_alloc_pe(phb);
975 if (pe_num == IODA_INVALID_PE) {
976 pr_warning("%s: Not enough PE# available, disabling device\n",
977 pci_name(dev));
978 return NULL;
979 }
980
981 /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
982 * pointer in the PE data structure, both should be destroyed at the
983 * same time. However, this needs to be looked at more closely again
984 * once we actually start removing things (Hotplug, SR-IOV, ...)
985 *
986 * At some point we want to remove the PDN completely anyways
987 */
988 pe = &phb->ioda.pe_array[pe_num];
989 pci_dev_get(dev);
990 pdn->pcidev = dev;
991 pdn->pe_number = pe_num;
992 pe->pdev = dev;
993 pe->pbus = NULL;
994 pe->tce32_seg = -1;
995 pe->mve_number = -1;
996 pe->rid = dev->bus->number << 8 | pdn->devfn;
997
998 pe_info(pe, "Associated device to PE\n");
999
1000 if (pnv_ioda_configure_pe(phb, pe)) {
1001 /* XXX What do we do here ? */
1002 if (pe_num)
1003 pnv_ioda_free_pe(phb, pe_num);
1004 pdn->pe_number = IODA_INVALID_PE;
1005 pe->pdev = NULL;
1006 pci_dev_put(dev);
1007 return NULL;
1008 }
1009
1010 /* Assign a DMA weight to the device */
1011 pe->dma_weight = pnv_ioda_dma_weight(dev);
1012 if (pe->dma_weight != 0) {
1013 phb->ioda.dma_weight += pe->dma_weight;
1014 phb->ioda.dma_pe_count++;
1015 }
1016
1017 /* Link the PE */
1018 pnv_ioda_link_pe_by_weight(phb, pe);
1019
1020 return pe;
1021}
fb446ad0 1022#endif /* Useful for SRIOV case */
184cd4a3
BH		 1023
1024static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
1025{
1026 struct pci_dev *dev;
1027
1028 list_for_each_entry(dev, &bus->devices, bus_list) {
b72c1f65 1029 struct pci_dn *pdn = pci_get_pdn(dev);
184cd4a3
BH		 1030
1031 if (pdn == NULL) {
1032 pr_warn("%s: No device node associated with device !\n",
1033 pci_name(dev));
1034 continue;
1035 }
184cd4a3
BH		 1036 pdn->pe_number = pe->pe_number;
1037 pe->dma_weight += pnv_ioda_dma_weight(dev);
fb446ad0 1038 if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
184cd4a3
BH		 1039 pnv_ioda_setup_same_PE(dev->subordinate, pe);
1040 }
1041}
1042
fb446ad0
GS		 1043/*
1044 * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1045 * single PCI bus. Another one that contains the primary PCI bus and its
1046 * subordinate PCI devices and buses. The second type of PE is normally
1047 * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1048 */
cad5cef6 1049static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
184cd4a3 1050{
fb446ad0 1051 struct pci_controller *hose = pci_bus_to_host(bus);
184cd4a3 1052 struct pnv_phb *phb = hose->private_data;
184cd4a3 1053 struct pnv_ioda_pe *pe;
262af557
GC		 1054 int pe_num = IODA_INVALID_PE;
1055
1056 /* Check if PE is determined by M64 */
1057 if (phb->pick_m64_pe)
1058 pe_num = phb->pick_m64_pe(phb, bus, all);
1059
1060 /* The PE number isn't pinned by M64 */
1061 if (pe_num == IODA_INVALID_PE)
1062 pe_num = pnv_ioda_alloc_pe(phb);
184cd4a3 1063
184cd4a3 1064 if (pe_num == IODA_INVALID_PE) {
fb446ad0
GS		 1065 pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1066 __func__, pci_domain_nr(bus), bus->number);
184cd4a3
BH		 1067 return;
1068 }
1069
1070 pe = &phb->ioda.pe_array[pe_num];
262af557 1071 pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
184cd4a3
BH		 1072 pe->pbus = bus;
1073 pe->pdev = NULL;
1074 pe->tce32_seg = -1;
1075 pe->mve_number = -1;
b918c62e 1076 pe->rid = bus->busn_res.start << 8;
184cd4a3
BH		 1077 pe->dma_weight = 0;
1078
fb446ad0
GS		 1079 if (all)
1080 pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
1081 bus->busn_res.start, bus->busn_res.end, pe_num);
1082 else
1083 pe_info(pe, "Secondary bus %d associated with PE#%d\n",
1084 bus->busn_res.start, pe_num);
184cd4a3
BH		 1085
1086 if (pnv_ioda_configure_pe(phb, pe)) {
1087 /* XXX What do we do here ? */
1088 if (pe_num)
1089 pnv_ioda_free_pe(phb, pe_num);
1090 pe->pbus = NULL;
1091 return;
1092 }
1093
1094 /* Associate it with all child devices */
1095 pnv_ioda_setup_same_PE(bus, pe);
1096
7ebdf956
GS		 1097 /* Put PE to the list */
1098 list_add_tail(&pe->list, &phb->ioda.pe_list);
1099
184cd4a3
BH		 1100 /* Account for one DMA PE if at least one DMA capable device exist
1101 * below the bridge
1102 */
1103 if (pe->dma_weight != 0) {
1104 phb->ioda.dma_weight += pe->dma_weight;
1105 phb->ioda.dma_pe_count++;
1106 }
1107
1108 /* Link the PE */
1109 pnv_ioda_link_pe_by_weight(phb, pe);
1110}
1111
cad5cef6 1112static void pnv_ioda_setup_PEs(struct pci_bus *bus)
184cd4a3
BH		 1113{
1114 struct pci_dev *dev;
fb446ad0
GS		 1115
1116 pnv_ioda_setup_bus_PE(bus, 0);
184cd4a3
BH		 1117
1118 list_for_each_entry(dev, &bus->devices, bus_list) {
fb446ad0
GS		 1119 if (dev->subordinate) {
1120 if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE)
1121 pnv_ioda_setup_bus_PE(dev->subordinate, 1);
1122 else
1123 pnv_ioda_setup_PEs(dev->subordinate);
1124 }
1125 }
1126}
1127
1128/*
1129 * Configure PEs so that the downstream PCI buses and devices
1130 * could have their associated PE#. Unfortunately, we didn't
1131 * figure out the way to identify the PLX bridge yet. So we
1132 * simply put the PCI bus and the subordinate behind the root
1133 * port to PE# here. The game rule here is expected to be changed
1134 * as soon as we can detected PLX bridge correctly.
1135 */
cad5cef6 1136static void pnv_pci_ioda_setup_PEs(void)
fb446ad0
GS		 1137{
1138 struct pci_controller *hose, *tmp;
262af557 1139 struct pnv_phb *phb;
fb446ad0
GS		 1140
1141 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
262af557
GC		 1142 phb = hose->private_data;
1143
1144 /* M64 layout might affect PE allocation */
5ef73567
GS		 1145 if (phb->reserve_m64_pe)
1146 phb->reserve_m64_pe(phb);
262af557 1147
fb446ad0 1148 pnv_ioda_setup_PEs(hose->bus);
184cd4a3
BH		 1149 }
1150}
1151
a8b2f828 1152#ifdef CONFIG_PCI_IOV
781a868f
WY		 1153static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
1154{
1155 struct pci_bus *bus;
1156 struct pci_controller *hose;
1157 struct pnv_phb *phb;
1158 struct pci_dn *pdn;
02639b0e 1159 int i, j;
781a868f
WY		 1160
1161 bus = pdev->bus;
1162 hose = pci_bus_to_host(bus);
1163 phb = hose->private_data;
1164 pdn = pci_get_pdn(pdev);
1165
02639b0e
WY		 1166 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
1167 for (j = 0; j < M64_PER_IOV; j++) {
1168 if (pdn->m64_wins[i][j] == IODA_INVALID_M64)
1169 continue;
1170 opal_pci_phb_mmio_enable(phb->opal_id,
1171 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
1172 clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc);
1173 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1174 }
781a868f
WY		 1175
1176 return 0;
1177}
1178
02639b0e 1179static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
781a868f
WY		 1180{
1181 struct pci_bus *bus;
1182 struct pci_controller *hose;
1183 struct pnv_phb *phb;
1184 struct pci_dn *pdn;
1185 unsigned int win;
1186 struct resource *res;
02639b0e 1187 int i, j;
781a868f 1188 int64_t rc;
02639b0e
WY		 1189 int total_vfs;
1190 resource_size_t size, start;
1191 int pe_num;
1192 int vf_groups;
1193 int vf_per_group;
781a868f
WY		 1194
1195 bus = pdev->bus;
1196 hose = pci_bus_to_host(bus);
1197 phb = hose->private_data;
1198 pdn = pci_get_pdn(pdev);
02639b0e 1199 total_vfs = pci_sriov_get_totalvfs(pdev);
781a868f
WY		 1200
1201 /* Initialize the m64_wins to IODA_INVALID_M64 */
1202 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
02639b0e
WY		 1203 for (j = 0; j < M64_PER_IOV; j++)
1204 pdn->m64_wins[i][j] = IODA_INVALID_M64;
1205
1206 if (pdn->m64_per_iov == M64_PER_IOV) {
1207 vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV;
1208 vf_per_group = (num_vfs <= M64_PER_IOV)? 1:
1209 roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1210 } else {
1211 vf_groups = 1;
1212 vf_per_group = 1;
1213 }
781a868f
WY		 1214
1215 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
1216 res = &pdev->resource[i + PCI_IOV_RESOURCES];
1217 if (!res->flags || !res->parent)
1218 continue;
1219
1220 if (!pnv_pci_is_mem_pref_64(res->flags))
1221 continue;
1222
02639b0e
WY		 1223 for (j = 0; j < vf_groups; j++) {
1224 do {
1225 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
1226 phb->ioda.m64_bar_idx + 1, 0);
1227
1228 if (win >= phb->ioda.m64_bar_idx + 1)
1229 goto m64_failed;
1230 } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
1231
1232 pdn->m64_wins[i][j] = win;
1233
1234 if (pdn->m64_per_iov == M64_PER_IOV) {
1235 size = pci_iov_resource_size(pdev,
1236 PCI_IOV_RESOURCES + i);
1237 size = size * vf_per_group;
1238 start = res->start + size * j;
1239 } else {
1240 size = resource_size(res);
1241 start = res->start;
1242 }
1243
1244 /* Map the M64 here */
1245 if (pdn->m64_per_iov == M64_PER_IOV) {
1246 pe_num = pdn->offset + j;
1247 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
1248 pe_num, OPAL_M64_WINDOW_TYPE,
1249 pdn->m64_wins[i][j], 0);
1250 }
1251
1252 rc = opal_pci_set_phb_mem_window(phb->opal_id,
1253 OPAL_M64_WINDOW_TYPE,
1254 pdn->m64_wins[i][j],
1255 start,
1256 0, /* unused */
1257 size);
781a868f 1258
781a868f 1259
02639b0e
WY		 1260 if (rc != OPAL_SUCCESS) {
1261 dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
1262 win, rc);
1263 goto m64_failed;
1264 }
781a868f 1265
02639b0e
WY		 1266 if (pdn->m64_per_iov == M64_PER_IOV)
1267 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1268 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
1269 else
1270 rc = opal_pci_phb_mmio_enable(phb->opal_id,
1271 OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
781a868f 1272
02639b0e
WY		 1273 if (rc != OPAL_SUCCESS) {
1274 dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
1275 win, rc);
1276 goto m64_failed;
1277 }
781a868f
WY		 1278 }
1279 }
1280 return 0;
1281
1282m64_failed:
1283 pnv_pci_vf_release_m64(pdev);
1284 return -EBUSY;
1285}
1286
1287static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
1288{
1289 struct pci_bus *bus;
1290 struct pci_controller *hose;
1291 struct pnv_phb *phb;
1292 struct iommu_table *tbl;
1293 unsigned long addr;
1294 int64_t rc;
1295
1296 bus = dev->bus;
1297 hose = pci_bus_to_host(bus);
1298 phb = hose->private_data;
b348aa65 1299 tbl = pe->table_group.tables[0];
781a868f
WY		 1300 addr = tbl->it_base;
1301
1302 opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1303 pe->pe_number << 1, 1, __pa(addr),
1304 0, 0x1000);
1305
1306 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1307 pe->pe_number,
1308 (pe->pe_number << 1) + 1,
1309 pe->tce_bypass_base,
1310 0);
1311 if (rc)
1312 pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
1313
0eaf4def
AK		 1314 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1315 if (pe->table_group.group) {
1316 iommu_group_put(pe->table_group.group);
1317 BUG_ON(pe->table_group.group);
ac9a5889 1318 }
aca6913f 1319 pnv_pci_ioda2_table_free_pages(tbl);
781a868f 1320 iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
781a868f
WY		 1321}
1322
02639b0e 1323static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
781a868f
WY		 1324{
1325 struct pci_bus *bus;
1326 struct pci_controller *hose;
1327 struct pnv_phb *phb;
1328 struct pnv_ioda_pe *pe, *pe_n;
1329 struct pci_dn *pdn;
02639b0e
WY		 1330 u16 vf_index;
1331 int64_t rc;
781a868f
WY		 1332
1333 bus = pdev->bus;
1334 hose = pci_bus_to_host(bus);
1335 phb = hose->private_data;
02639b0e 1336 pdn = pci_get_pdn(pdev);
781a868f
WY		 1337
1338 if (!pdev->is_physfn)
1339 return;
1340
02639b0e
WY		 1341 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1342 int vf_group;
1343 int vf_per_group;
1344 int vf_index1;
1345
1346 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1347
1348 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
1349 for (vf_index = vf_group * vf_per_group;
1350 vf_index < (vf_group + 1) * vf_per_group &&
1351 vf_index < num_vfs;
1352 vf_index++)
1353 for (vf_index1 = vf_group * vf_per_group;
1354 vf_index1 < (vf_group + 1) * vf_per_group &&
1355 vf_index1 < num_vfs;
1356 vf_index1++){
1357
1358 rc = opal_pci_set_peltv(phb->opal_id,
1359 pdn->offset + vf_index,
1360 pdn->offset + vf_index1,
1361 OPAL_REMOVE_PE_FROM_DOMAIN);
1362
1363 if (rc)
1364 dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n",
1365 __func__,
1366 pdn->offset + vf_index1, rc);
1367 }
1368 }
1369
781a868f
WY		 1370 list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
1371 if (pe->parent_dev != pdev)
1372 continue;
1373
1374 pnv_pci_ioda2_release_dma_pe(pdev, pe);
1375
1376 /* Remove from list */
1377 mutex_lock(&phb->ioda.pe_list_mutex);
1378 list_del(&pe->list);
1379 mutex_unlock(&phb->ioda.pe_list_mutex);
1380
1381 pnv_ioda_deconfigure_pe(phb, pe);
1382
1383 pnv_ioda_free_pe(phb, pe->pe_number);
1384 }
1385}
1386
1387void pnv_pci_sriov_disable(struct pci_dev *pdev)
1388{
1389 struct pci_bus *bus;
1390 struct pci_controller *hose;
1391 struct pnv_phb *phb;
1392 struct pci_dn *pdn;
1393 struct pci_sriov *iov;
1394 u16 num_vfs;
1395
1396 bus = pdev->bus;
1397 hose = pci_bus_to_host(bus);
1398 phb = hose->private_data;
1399 pdn = pci_get_pdn(pdev);
1400 iov = pdev->sriov;
1401 num_vfs = pdn->num_vfs;
1402
1403 /* Release VF PEs */
02639b0e 1404 pnv_ioda_release_vf_PE(pdev, num_vfs);
781a868f
WY		 1405
1406 if (phb->type == PNV_PHB_IODA2) {
02639b0e
WY		 1407 if (pdn->m64_per_iov == 1)
1408 pnv_pci_vf_resource_shift(pdev, -pdn->offset);
781a868f
WY		 1409
1410 /* Release M64 windows */
1411 pnv_pci_vf_release_m64(pdev);
1412
1413 /* Release PE numbers */
1414 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1415 pdn->offset = 0;
1416 }
1417}
1418
1419static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1420 struct pnv_ioda_pe *pe);
1421static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
1422{
1423 struct pci_bus *bus;
1424 struct pci_controller *hose;
1425 struct pnv_phb *phb;
1426 struct pnv_ioda_pe *pe;
1427 int pe_num;
1428 u16 vf_index;
1429 struct pci_dn *pdn;
02639b0e 1430 int64_t rc;
781a868f
WY		 1431
1432 bus = pdev->bus;
1433 hose = pci_bus_to_host(bus);
1434 phb = hose->private_data;
1435 pdn = pci_get_pdn(pdev);
1436
1437 if (!pdev->is_physfn)
1438 return;
1439
1440 /* Reserve PE for each VF */
1441 for (vf_index = 0; vf_index < num_vfs; vf_index++) {
1442 pe_num = pdn->offset + vf_index;
1443
1444 pe = &phb->ioda.pe_array[pe_num];
1445 pe->pe_number = pe_num;
1446 pe->phb = phb;
1447 pe->flags = PNV_IODA_PE_VF;
1448 pe->pbus = NULL;
1449 pe->parent_dev = pdev;
1450 pe->tce32_seg = -1;
1451 pe->mve_number = -1;
1452 pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
1453 pci_iov_virtfn_devfn(pdev, vf_index);
1454
1455 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
1456 hose->global_number, pdev->bus->number,
1457 PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
1458 PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
1459
1460 if (pnv_ioda_configure_pe(phb, pe)) {
1461 /* XXX What do we do here ? */
1462 if (pe_num)
1463 pnv_ioda_free_pe(phb, pe_num);
1464 pe->pdev = NULL;
1465 continue;
1466 }
1467
781a868f
WY		 1468 /* Put PE to the list */
1469 mutex_lock(&phb->ioda.pe_list_mutex);
1470 list_add_tail(&pe->list, &phb->ioda.pe_list);
1471 mutex_unlock(&phb->ioda.pe_list_mutex);
1472
1473 pnv_pci_ioda2_setup_dma_pe(phb, pe);
1474 }
02639b0e
WY		 1475
1476 if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) {
1477 int vf_group;
1478 int vf_per_group;
1479 int vf_index1;
1480
1481 vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov;
1482
1483 for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) {
1484 for (vf_index = vf_group * vf_per_group;
1485 vf_index < (vf_group + 1) * vf_per_group &&
1486 vf_index < num_vfs;
1487 vf_index++) {
1488 for (vf_index1 = vf_group * vf_per_group;
1489 vf_index1 < (vf_group + 1) * vf_per_group &&
1490 vf_index1 < num_vfs;
1491 vf_index1++) {
1492
1493 rc = opal_pci_set_peltv(phb->opal_id,
1494 pdn->offset + vf_index,
1495 pdn->offset + vf_index1,
1496 OPAL_ADD_PE_TO_DOMAIN);
1497
1498 if (rc)
1499 dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n",
1500 __func__,
1501 pdn->offset + vf_index1, rc);
1502 }
1503 }
1504 }
1505 }
781a868f
WY		 1506}
1507
1508int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1509{
1510 struct pci_bus *bus;
1511 struct pci_controller *hose;
1512 struct pnv_phb *phb;
1513 struct pci_dn *pdn;
1514 int ret;
1515
1516 bus = pdev->bus;
1517 hose = pci_bus_to_host(bus);
1518 phb = hose->private_data;
1519 pdn = pci_get_pdn(pdev);
1520
1521 if (phb->type == PNV_PHB_IODA2) {
1522 /* Calculate available PE for required VFs */
1523 mutex_lock(&phb->ioda.pe_alloc_mutex);
1524 pdn->offset = bitmap_find_next_zero_area(
1525 phb->ioda.pe_alloc, phb->ioda.total_pe,
1526 0, num_vfs, 0);
1527 if (pdn->offset >= phb->ioda.total_pe) {
1528 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1529 dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
1530 pdn->offset = 0;
1531 return -EBUSY;
1532 }
1533 bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1534 pdn->num_vfs = num_vfs;
1535 mutex_unlock(&phb->ioda.pe_alloc_mutex);
1536
1537 /* Assign M64 window accordingly */
02639b0e 1538 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
781a868f
WY		 1539 if (ret) {
1540 dev_info(&pdev->dev, "Not enough M64 window resources\n");
1541 goto m64_failed;
1542 }
1543
1544 /*
1545 * When using one M64 BAR to map one IOV BAR, we need to shift
1546 * the IOV BAR according to the PE# allocated to the VFs.
1547 * Otherwise, the PE# for the VF will conflict with others.
1548 */
02639b0e
WY		 1549 if (pdn->m64_per_iov == 1) {
1550 ret = pnv_pci_vf_resource_shift(pdev, pdn->offset);
1551 if (ret)
1552 goto m64_failed;
1553 }
781a868f
WY		 1554 }
1555
1556 /* Setup VF PEs */
1557 pnv_ioda_setup_vf_PE(pdev, num_vfs);
1558
1559 return 0;
1560
1561m64_failed:
1562 bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs);
1563 pdn->offset = 0;
1564
1565 return ret;
1566}
1567
a8b2f828
GS		 1568int pcibios_sriov_disable(struct pci_dev *pdev)
1569{
781a868f
WY		 1570 pnv_pci_sriov_disable(pdev);
1571
a8b2f828
GS		 1572 /* Release PCI data */
1573 remove_dev_pci_data(pdev);
1574 return 0;
1575}
1576
1577int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
1578{
1579 /* Allocate PCI data */
1580 add_dev_pci_data(pdev);
781a868f
WY		 1581
1582 pnv_pci_sriov_enable(pdev, num_vfs);
a8b2f828
GS		 1583 return 0;
1584}
1585#endif /* CONFIG_PCI_IOV */
1586
959c9bdd 1587static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
184cd4a3 1588{
b72c1f65 1589 struct pci_dn *pdn = pci_get_pdn(pdev);
959c9bdd 1590 struct pnv_ioda_pe *pe;
184cd4a3 1591
959c9bdd
GS		 1592 /*
1593 * The function can be called while the PE#
1594 * hasn't been assigned. Do nothing for the
1595 * case.
1596 */
1597 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
1598 return;
184cd4a3 1599
959c9bdd 1600 pe = &phb->ioda.pe_array[pdn->pe_number];
cd15b048 1601 WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
b348aa65 1602 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
4617082e
AK		 1603 /*
1604 * Note: iommu_add_device() will fail here as
1605 * for physical PE: the device is already added by now;
1606 * for virtual PE: sysfs entries are not ready yet and
1607 * tce_iommu_bus_notifier will add the device to a group later.
1608 */
184cd4a3
BH		 1609}
1610
763d2d8d 1611static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
cd15b048 1612{
763d2d8d
DA		 1613 struct pci_controller *hose = pci_bus_to_host(pdev->bus);
1614 struct pnv_phb *phb = hose->private_data;
cd15b048
BH		 1615 struct pci_dn *pdn = pci_get_pdn(pdev);
1616 struct pnv_ioda_pe *pe;
1617 uint64_t top;
1618 bool bypass = false;
1619
1620 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1621 return -ENODEV;;
1622
1623 pe = &phb->ioda.pe_array[pdn->pe_number];
1624 if (pe->tce_bypass_enabled) {
1625 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1626 bypass = (dma_mask >= top);
1627 }
1628
1629 if (bypass) {
1630 dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
1631 set_dma_ops(&pdev->dev, &dma_direct_ops);
1632 set_dma_offset(&pdev->dev, pe->tce_bypass_base);
1633 } else {
1634 dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
1635 set_dma_ops(&pdev->dev, &dma_iommu_ops);
b348aa65 1636 set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
cd15b048 1637 }
a32305bf 1638 *pdev->dev.dma_mask = dma_mask;
cd15b048
BH		 1639 return 0;
1640}
1641
fe7e85c6
GS		 1642static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb,
1643 struct pci_dev *pdev)
1644{
1645 struct pci_dn *pdn = pci_get_pdn(pdev);
1646 struct pnv_ioda_pe *pe;
1647 u64 end, mask;
1648
1649 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1650 return 0;
1651
1652 pe = &phb->ioda.pe_array[pdn->pe_number];
1653 if (!pe->tce_bypass_enabled)
1654 return __dma_get_required_mask(&pdev->dev);
1655
1656
1657 end = pe->tce_bypass_base + memblock_end_of_DRAM();
1658 mask = 1ULL << (fls64(end) - 1);
1659 mask += mask - 1;
1660
1661 return mask;
1662}
1663
dff4a39e 1664static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
ea30e99e 1665 struct pci_bus *bus)
74251fe2
BH		 1666{
1667 struct pci_dev *dev;
1668
1669 list_for_each_entry(dev, &bus->devices, bus_list) {
b348aa65 1670 set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
4617082e 1671 iommu_add_device(&dev->dev);
dff4a39e 1672
74251fe2 1673 if (dev->subordinate)
ea30e99e 1674 pnv_ioda_setup_bus_dma(pe, dev->subordinate);
74251fe2
BH		 1675 }
1676}
1677
decbda25
AK		 1678static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
1679 unsigned long index, unsigned long npages, bool rm)
4cce9550 1680{
0eaf4def
AK		 1681 struct iommu_table_group_link *tgl = list_first_entry_or_null(
1682 &tbl->it_group_list, struct iommu_table_group_link,
1683 next);
1684 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
b348aa65 1685 struct pnv_ioda_pe, table_group);
3ad26e5c 1686 __be64 __iomem *invalidate = rm ?
5780fb04
AK		 1687 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1688 pe->phb->ioda.tce_inval_reg;
4cce9550 1689 unsigned long start, end, inc;
b0376c9b 1690 const unsigned shift = tbl->it_page_shift;
4cce9550 1691
decbda25
AK		 1692 start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1693 end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1694 npages - 1);
4cce9550
GS		 1695
1696 /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
1697 if (tbl->it_busno) {
b0376c9b
AK		 1698 start <<= shift;
1699 end <<= shift;
1700 inc = 128ull << shift;
4cce9550
GS		 1701 start |= tbl->it_busno;
1702 end |= tbl->it_busno;
1703 } else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
1704 /* p7ioc-style invalidation, 2 TCEs per write */
1705 start |= (1ull << 63);
1706 end |= (1ull << 63);
1707 inc = 16;
1708 } else {
1709 /* Default (older HW) */
1710 inc = 128;
1711 }
1712
1713 end |= inc - 1; /* round up end to be different than start */
1714
1715 mb(); /* Ensure above stores are visible */
1716 while (start <= end) {
8e0a1611 1717 if (rm)
3ad26e5c 1718 __raw_rm_writeq(cpu_to_be64(start), invalidate);
8e0a1611 1719 else
3ad26e5c 1720 __raw_writeq(cpu_to_be64(start), invalidate);
4cce9550
GS		 1721 start += inc;
1722 }
1723
1724 /*
1725 * The iommu layer will do another mb() for us on build()
1726 * and we don't care on free()
1727 */
1728}
1729
decbda25
AK		 1730static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1731 long npages, unsigned long uaddr,
1732 enum dma_data_direction direction,
1733 struct dma_attrs *attrs)
1734{
1735 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1736 attrs);
1737
1738 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1739 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1740
1741 return ret;
1742}
1743
05c6cfb9
AK		 1744#ifdef CONFIG_IOMMU_API
1745static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
1746 unsigned long *hpa, enum dma_data_direction *direction)
1747{
1748 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1749
1750 if (!ret && (tbl->it_type &
1751 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1752 pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
1753
1754 return ret;
1755}
1756#endif
1757
decbda25
AK		 1758static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1759 long npages)
1760{
1761 pnv_tce_free(tbl, index, npages);
1762
1763 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1764 pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);
1765}
1766
da004c36 1767static struct iommu_table_ops pnv_ioda1_iommu_ops = {
decbda25 1768 .set = pnv_ioda1_tce_build,
05c6cfb9
AK		 1769#ifdef CONFIG_IOMMU_API
1770 .exchange = pnv_ioda1_tce_xchg,
1771#endif
decbda25 1772 .clear = pnv_ioda1_tce_free,
da004c36
AK		 1773 .get = pnv_tce_get,
1774};
1775
5780fb04
AK		 1776static inline void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_ioda_pe *pe)
1777{
1778 /* 01xb - invalidate TCEs that match the specified PE# */
1779 unsigned long val = (0x4ull << 60) | (pe->pe_number & 0xFF);
1780 struct pnv_phb *phb = pe->phb;
1781
1782 if (!phb->ioda.tce_inval_reg)
1783 return;
1784
1785 mb(); /* Ensure above stores are visible */
1786 __raw_writeq(cpu_to_be64(val), phb->ioda.tce_inval_reg);
1787}
1788
e57080f1
AK		 1789static void pnv_pci_ioda2_do_tce_invalidate(unsigned pe_number, bool rm,
1790 __be64 __iomem *invalidate, unsigned shift,
1791 unsigned long index, unsigned long npages)
4cce9550
GS		 1792{
1793 unsigned long start, end, inc;
4cce9550
GS		 1794
1795 /* We'll invalidate DMA address in PE scope */
b0376c9b 1796 start = 0x2ull << 60;
e57080f1 1797 start |= (pe_number & 0xFF);
4cce9550
GS		 1798 end = start;
1799
1800 /* Figure out the start, end and step */
decbda25
AK		 1801 start |= (index << shift);
1802 end |= ((index + npages - 1) << shift);
b0376c9b 1803 inc = (0x1ull << shift);
4cce9550
GS		 1804 mb();
1805
1806 while (start <= end) {
8e0a1611 1807 if (rm)
3ad26e5c 1808 __raw_rm_writeq(cpu_to_be64(start), invalidate);
8e0a1611 1809 else
3ad26e5c 1810 __raw_writeq(cpu_to_be64(start), invalidate);
4cce9550
GS		 1811 start += inc;
1812 }
1813}
1814
e57080f1
AK		 1815static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1816 unsigned long index, unsigned long npages, bool rm)
1817{
1818 struct iommu_table_group_link *tgl;
1819
1820 list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
1821 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1822 struct pnv_ioda_pe, table_group);
1823 __be64 __iomem *invalidate = rm ?
1824 (__be64 __iomem *)pe->phb->ioda.tce_inval_reg_phys :
1825 pe->phb->ioda.tce_inval_reg;
1826
1827 pnv_pci_ioda2_do_tce_invalidate(pe->pe_number, rm,
1828 invalidate, tbl->it_page_shift,
1829 index, npages);
1830 }
1831}
1832
decbda25
AK		 1833static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1834 long npages, unsigned long uaddr,
1835 enum dma_data_direction direction,
1836 struct dma_attrs *attrs)
4cce9550 1837{
decbda25
AK		 1838 int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1839 attrs);
4cce9550 1840
decbda25
AK		 1841 if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))
1842 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
1843
1844 return ret;
1845}
1846
05c6cfb9
AK		 1847#ifdef CONFIG_IOMMU_API
1848static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
1849 unsigned long *hpa, enum dma_data_direction *direction)
1850{
1851 long ret = pnv_tce_xchg(tbl, index, hpa, direction);
1852
1853 if (!ret && (tbl->it_type &
1854 (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
1855 pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
1856
1857 return ret;
1858}
1859#endif
1860
decbda25
AK		 1861static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1862 long npages)
1863{
1864 pnv_tce_free(tbl, index, npages);
1865
1866 if (tbl->it_type & TCE_PCI_SWINV_FREE)
1867 pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
4cce9550
GS		 1868}
1869
da004c36 1870static struct iommu_table_ops pnv_ioda2_iommu_ops = {
decbda25 1871 .set = pnv_ioda2_tce_build,
05c6cfb9
AK		 1872#ifdef CONFIG_IOMMU_API
1873 .exchange = pnv_ioda2_tce_xchg,
1874#endif
decbda25 1875 .clear = pnv_ioda2_tce_free,
da004c36
AK		 1876 .get = pnv_tce_get,
1877};
1878
cad5cef6
GKH		 1879static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
1880 struct pnv_ioda_pe *pe, unsigned int base,
1881 unsigned int segs)
184cd4a3
BH		 1882{
1883
1884 struct page *tce_mem = NULL;
184cd4a3
BH		 1885 struct iommu_table *tbl;
1886 unsigned int i;
1887 int64_t rc;
1888 void *addr;
1889
184cd4a3
BH		 1890 /* XXX FIXME: Handle 64-bit only DMA devices */
1891 /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1892 /* XXX FIXME: Allocate multi-level tables on PHB3 */
1893
1894 /* We shouldn't already have a 32-bit DMA associated */
1895 if (WARN_ON(pe->tce32_seg >= 0))
1896 return;
1897
0eaf4def 1898 tbl = pnv_pci_table_alloc(phb->hose->node);
b348aa65
AK		 1899 iommu_register_group(&pe->table_group, phb->hose->global_number,
1900 pe->pe_number);
0eaf4def 1901 pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
c5773822 1902
184cd4a3
BH		 1903 /* Grab a 32-bit TCE table */
1904 pe->tce32_seg = base;
1905 pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1906 (base << 28), ((base + segs) << 28) - 1);
1907
1908 /* XXX Currently, we allocate one big contiguous table for the
1909 * TCEs. We only really need one chunk per 256M of TCE space
1910 * (ie per segment) but that's an optimization for later, it
1911 * requires some added smarts with our get/put_tce implementation
1912 */
1913 tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1914 get_order(TCE32_TABLE_SIZE * segs));
1915 if (!tce_mem) {
1916 pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1917 goto fail;
1918 }
1919 addr = page_address(tce_mem);
1920 memset(addr, 0, TCE32_TABLE_SIZE * segs);
1921
1922 /* Configure HW */
1923 for (i = 0; i < segs; i++) {
1924 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1925 pe->pe_number,
1926 base + i, 1,
1927 __pa(addr) + TCE32_TABLE_SIZE * i,
1928 TCE32_TABLE_SIZE, 0x1000);
1929 if (rc) {
1930 pe_err(pe, " Failed to configure 32-bit TCE table,"
1931 " err %ld\n", rc);
1932 goto fail;
1933 }
1934 }
1935
1936 /* Setup linux iommu table */
184cd4a3 1937 pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
8fa5d454 1938 base << 28, IOMMU_PAGE_SHIFT_4K);
184cd4a3
BH		 1939
1940 /* OPAL variant of P7IOC SW invalidated TCEs */
5780fb04 1941 if (phb->ioda.tce_inval_reg)
65fd766b
GS		 1942 tbl->it_type |= (TCE_PCI_SWINV_CREATE |
1943 TCE_PCI_SWINV_FREE |
1944 TCE_PCI_SWINV_PAIR);
5780fb04 1945
da004c36 1946 tbl->it_ops = &pnv_ioda1_iommu_ops;
184cd4a3
BH		 1947 iommu_init_table(tbl, phb->hose->node);
1948
781a868f 1949 if (pe->flags & PNV_IODA_PE_DEV) {
4617082e
AK		 1950 /*
1951 * Setting table base here only for carrying iommu_group
1952 * further down to let iommu_add_device() do the job.
1953 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
1954 */
1955 set_iommu_table_base(&pe->pdev->dev, tbl);
1956 iommu_add_device(&pe->pdev->dev);
c5773822 1957 } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
ea30e99e 1958 pnv_ioda_setup_bus_dma(pe, pe->pbus);
74251fe2 1959
184cd4a3
BH		 1960 return;
1961 fail:
1962 /* XXX Failure: Try to fallback to 64-bit only ? */
1963 if (pe->tce32_seg >= 0)
1964 pe->tce32_seg = -1;
1965 if (tce_mem)
1966 __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
0eaf4def
AK		 1967 if (tbl) {
1968 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1969 iommu_free_table(tbl, "pnv");
1970 }
184cd4a3
BH		 1971}
1972
43cb60ab
AK		 1973static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
1974 int num, struct iommu_table *tbl)
1975{
1976 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1977 table_group);
1978 struct pnv_phb *phb = pe->phb;
1979 int64_t rc;
1980 const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
1981 const __u64 win_size = tbl->it_size << tbl->it_page_shift;
1982
1983 pe_info(pe, "Setting up window %llx..%llx pg=%x\n",
1984 start_addr, start_addr + win_size - 1,
1985 IOMMU_PAGE_SIZE(tbl));
1986
1987 /*
1988 * Map TCE table through TVT. The TVE index is the PE number
1989 * shifted by 1 bit for 32-bits DMA space.
1990 */
1991 rc = opal_pci_map_pe_dma_window(phb->opal_id,
1992 pe->pe_number,
1993 pe->pe_number << 1,
1994 1,
1995 __pa(tbl->it_base),
1996 tbl->it_size << 3,
1997 IOMMU_PAGE_SIZE(tbl));
1998 if (rc) {
1999 pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
2000 return rc;
2001 }
2002
2003 pnv_pci_link_table_and_group(phb->hose->node, num,
2004 tbl, &pe->table_group);
2005 pnv_pci_ioda2_tce_invalidate_entire(pe);
2006
2007 return 0;
2008}
2009
f87a8864 2010static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
cd15b048 2011{
cd15b048
BH		 2012 uint16_t window_id = (pe->pe_number << 1 ) + 1;
2013 int64_t rc;
2014
2015 pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
2016 if (enable) {
2017 phys_addr_t top = memblock_end_of_DRAM();
2018
2019 top = roundup_pow_of_two(top);
2020 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2021 pe->pe_number,
2022 window_id,
2023 pe->tce_bypass_base,
2024 top);
2025 } else {
2026 rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
2027 pe->pe_number,
2028 window_id,
2029 pe->tce_bypass_base,
2030 0);
cd15b048
BH		 2031 }
2032 if (rc)
2033 pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
2034 else
2035 pe->tce_bypass_enabled = enable;
2036}
2037
f87a8864
AK		 2038#ifdef CONFIG_IOMMU_API
2039static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
cd15b048 2040{
f87a8864
AK		 2041 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2042 table_group);
cd15b048 2043
f87a8864
AK		 2044 iommu_take_ownership(table_group->tables[0]);
2045 pnv_pci_ioda2_set_bypass(pe, false);
2046}
cd15b048 2047
f87a8864
AK		 2048static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
2049{
2050 struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
2051 table_group);
2052
2053 iommu_release_ownership(table_group->tables[0]);
2054 pnv_pci_ioda2_set_bypass(pe, true);
cd15b048
BH		 2055}
2056
f87a8864
AK		 2057static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
2058 .take_ownership = pnv_ioda2_take_ownership,
2059 .release_ownership = pnv_ioda2_release_ownership,
2060};
2061#endif
2062
5780fb04
AK		 2063static void pnv_pci_ioda_setup_opal_tce_kill(struct pnv_phb *phb)
2064{
2065 const __be64 *swinvp;
2066
2067 /* OPAL variant of PHB3 invalidated TCEs */
2068 swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
2069 if (!swinvp)
2070 return;
2071
2072 phb->ioda.tce_inval_reg_phys = be64_to_cpup(swinvp);
2073 phb->ioda.tce_inval_reg = ioremap(phb->ioda.tce_inval_reg_phys, 8);
2074}
2075
aca6913f 2076static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift)
373f5657
GS		 2077{
2078 struct page *tce_mem = NULL;
aca6913f
AK		 2079 __be64 *addr;
2080 unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
2081
2082 tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
2083 if (!tce_mem) {
2084 pr_err("Failed to allocate a TCE memory, order=%d\n", order);
2085 return NULL;
2086 }
2087 addr = page_address(tce_mem);
2088 memset(addr, 0, 1UL << (order + PAGE_SHIFT));
2089
2090 return addr;
2091}
2092
2093static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
2094 __u32 page_shift, __u64 window_size, struct iommu_table *tbl)
2095{
373f5657 2096 void *addr;
aca6913f
AK		 2097 const unsigned window_shift = ilog2(window_size);
2098 unsigned entries_shift = window_shift - page_shift;
2099 unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT);
2100 const unsigned long tce_table_size = 1UL << table_shift;
2101
2102 if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
2103 return -EINVAL;
2104
2105 /* Allocate TCE table */
2106 addr = pnv_pci_ioda2_table_do_alloc_pages(nid, table_shift);
2107 if (!addr)
2108 return -ENOMEM;
2109
2110 /* Setup linux iommu table */
2111 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
2112 page_shift);
2113
2114 pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
2115 window_size, tce_table_size, bus_offset);
2116
2117 return 0;
2118}
2119
2120static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
2121{
2122 if (!tbl->it_size)
2123 return;
2124
2125 free_pages(tbl->it_base, get_order(tbl->it_size << 3));
2126}
2127
2128static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
2129 struct pnv_ioda_pe *pe)
2130{
373f5657 2131 struct iommu_table *tbl;
373f5657
GS		 2132 int64_t rc;
2133
2134 /* We shouldn't already have a 32-bit DMA associated */
2135 if (WARN_ON(pe->tce32_seg >= 0))
2136 return;
2137
f87a8864
AK		 2138 /* TVE #1 is selected by PCI address bit 59 */
2139 pe->tce_bypass_base = 1ull << 59;
2140
0eaf4def 2141 tbl = pnv_pci_table_alloc(phb->hose->node);
b348aa65
AK		 2142 iommu_register_group(&pe->table_group, phb->hose->global_number,
2143 pe->pe_number);
0eaf4def 2144 pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
c5773822 2145
373f5657
GS		 2146 /* The PE will reserve all possible 32-bits space */
2147 pe->tce32_seg = 0;
373f5657 2148 pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
aca6913f 2149 phb->ioda.m32_pci_base);
373f5657 2150
aca6913f
AK		 2151 /* Setup linux iommu table */
2152 rc = pnv_pci_ioda2_table_alloc_pages(pe->phb->hose->node,
2153 0, IOMMU_PAGE_SHIFT_4K, phb->ioda.m32_pci_base, tbl);
2154 if (rc) {
2155 pe_err(pe, "Failed to create 32-bit TCE table, err %ld", rc);
373f5657
GS		 2156 goto fail;
2157 }
e5aad1e6
AK		 2158
2159 tbl->it_ops = &pnv_ioda2_iommu_ops;
2160 iommu_init_table(tbl, phb->hose->node);
2161#ifdef CONFIG_IOMMU_API
2162 pe->table_group.ops = &pnv_pci_ioda2_ops;
2163#endif
2164
43cb60ab 2165 rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
373f5657
GS		 2166 if (rc) {
2167 pe_err(pe, "Failed to configure 32-bit TCE table,"
2168 " err %ld\n", rc);
2169 goto fail;
2170 }
2171
373f5657 2172 /* OPAL variant of PHB3 invalidated TCEs */
5780fb04 2173 if (phb->ioda.tce_inval_reg)
65fd766b 2174 tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);
5780fb04 2175
781a868f 2176 if (pe->flags & PNV_IODA_PE_DEV) {
4617082e
AK		 2177 /*
2178 * Setting table base here only for carrying iommu_group
2179 * further down to let iommu_add_device() do the job.
2180 * pnv_pci_ioda_dma_dev_setup will override it later anyway.
2181 */
2182 set_iommu_table_base(&pe->pdev->dev, tbl);
2183 iommu_add_device(&pe->pdev->dev);
c5773822 2184 } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
ea30e99e 2185 pnv_ioda_setup_bus_dma(pe, pe->pbus);
74251fe2 2186
cd15b048 2187 /* Also create a bypass window */
4e287840 2188 if (!pnv_iommu_bypass_disabled)
f87a8864 2189 pnv_pci_ioda2_set_bypass(pe, true);
4e287840 2190
373f5657
GS		 2191 return;
2192fail:
2193 if (pe->tce32_seg >= 0)
2194 pe->tce32_seg = -1;
0eaf4def 2195 if (tbl) {
aca6913f 2196 pnv_pci_ioda2_table_free_pages(tbl);
0eaf4def
AK		 2197 pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
2198 iommu_free_table(tbl, "pnv");
2199 }
373f5657
GS		 2200}
2201
cad5cef6 2202static void pnv_ioda_setup_dma(struct pnv_phb *phb)
184cd4a3
BH		 2203{
2204 struct pci_controller *hose = phb->hose;
2205 unsigned int residual, remaining, segs, tw, base;
2206 struct pnv_ioda_pe *pe;
2207
2208 /* If we have more PE# than segments available, hand out one
2209 * per PE until we run out and let the rest fail. If not,
2210 * then we assign at least one segment per PE, plus more based
2211 * on the amount of devices under that PE
2212 */
2213 if (phb->ioda.dma_pe_count > phb->ioda.tce32_count)
2214 residual = 0;
2215 else
2216 residual = phb->ioda.tce32_count -
2217 phb->ioda.dma_pe_count;
2218
2219 pr_info("PCI: Domain %04x has %ld available 32-bit DMA segments\n",
2220 hose->global_number, phb->ioda.tce32_count);
2221 pr_info("PCI: %d PE# for a total weight of %d\n",
2222 phb->ioda.dma_pe_count, phb->ioda.dma_weight);
2223
5780fb04
AK		 2224 pnv_pci_ioda_setup_opal_tce_kill(phb);
2225
184cd4a3
BH		 2226 /* Walk our PE list and configure their DMA segments, hand them
2227 * out one base segment plus any residual segments based on
2228 * weight
2229 */
2230 remaining = phb->ioda.tce32_count;
2231 tw = phb->ioda.dma_weight;
2232 base = 0;
7ebdf956 2233 list_for_each_entry(pe, &phb->ioda.pe_dma_list, dma_link) {
184cd4a3
BH		 2234 if (!pe->dma_weight)
2235 continue;
2236 if (!remaining) {
2237 pe_warn(pe, "No DMA32 resources available\n");
2238 continue;
2239 }
2240 segs = 1;
2241 if (residual) {
2242 segs += ((pe->dma_weight * residual) + (tw / 2)) / tw;
2243 if (segs > remaining)
2244 segs = remaining;
2245 }
373f5657
GS		 2246
2247 /*
2248 * For IODA2 compliant PHB3, we needn't care about the weight.
2249 * The all available 32-bits DMA space will be assigned to
2250 * the specific PE.
2251 */
2252 if (phb->type == PNV_PHB_IODA1) {
2253 pe_info(pe, "DMA weight %d, assigned %d DMA32 segments\n",
2254 pe->dma_weight, segs);
2255 pnv_pci_ioda_setup_dma_pe(phb, pe, base, segs);
2256 } else {
2257 pe_info(pe, "Assign DMA32 space\n");
2258 segs = 0;
2259 pnv_pci_ioda2_setup_dma_pe(phb, pe);
2260 }
2261
184cd4a3
BH		 2262 remaining -= segs;
2263 base += segs;
2264 }
2265}
2266
2267#ifdef CONFIG_PCI_MSI
137436c9
GS		 2268static void pnv_ioda2_msi_eoi(struct irq_data *d)
2269{
2270 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
2271 struct irq_chip *chip = irq_data_get_irq_chip(d);
2272 struct pnv_phb *phb = container_of(chip, struct pnv_phb,
2273 ioda.irq_chip);
2274 int64_t rc;
2275
2276 rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
2277 WARN_ON_ONCE(rc);
2278
2279 icp_native_eoi(d);
2280}
2281
fd9a1c26
IM		 2282
2283static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
2284{
2285 struct irq_data *idata;
2286 struct irq_chip *ichip;
2287
2288 if (phb->type != PNV_PHB_IODA2)
2289 return;
2290
2291 if (!phb->ioda.irq_chip_init) {
2292 /*
2293 * First time we setup an MSI IRQ, we need to setup the
2294 * corresponding IRQ chip to route correctly.
2295 */
2296 idata = irq_get_irq_data(virq);
2297 ichip = irq_data_get_irq_chip(idata);
2298 phb->ioda.irq_chip_init = 1;
2299 phb->ioda.irq_chip = *ichip;
2300 phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2301 }
2302 irq_set_chip(virq, &phb->ioda.irq_chip);
2303}
2304
80c49c7e
IM		 2305#ifdef CONFIG_CXL_BASE
2306
6f963ec2 2307struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev)
80c49c7e
IM		 2308{
2309 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2310
6f963ec2 2311 return of_node_get(hose->dn);
80c49c7e 2312}
6f963ec2 2313EXPORT_SYMBOL(pnv_pci_get_phb_node);
80c49c7e 2314
1212aa1c 2315int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
80c49c7e
IM		 2316{
2317 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2318 struct pnv_phb *phb = hose->private_data;
2319 struct pnv_ioda_pe *pe;
2320 int rc;
2321
2322 pe = pnv_ioda_get_pe(dev);
2323 if (!pe)
2324 return -ENODEV;
2325
2326 pe_info(pe, "Switching PHB to CXL\n");
2327
1212aa1c 2328 rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
80c49c7e
IM		 2329 if (rc)
2330 dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
2331
2332 return rc;
2333}
1212aa1c 2334EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
80c49c7e
IM		 2335
2336/* Find PHB for cxl dev and allocate MSI hwirqs?
2337 * Returns the absolute hardware IRQ number
2338 */
2339int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num)
2340{
2341 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2342 struct pnv_phb *phb = hose->private_data;
2343 int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num);
2344
2345 if (hwirq < 0) {
2346 dev_warn(&dev->dev, "Failed to find a free MSI\n");
2347 return -ENOSPC;
2348 }
2349
2350 return phb->msi_base + hwirq;
2351}
2352EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs);
2353
2354void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num)
2355{
2356 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2357 struct pnv_phb *phb = hose->private_data;
2358
2359 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num);
2360}
2361EXPORT_SYMBOL(pnv_cxl_release_hwirqs);
2362
2363void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
2364 struct pci_dev *dev)
2365{
2366 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2367 struct pnv_phb *phb = hose->private_data;
2368 int i, hwirq;
2369
2370 for (i = 1; i < CXL_IRQ_RANGES; i++) {
2371 if (!irqs->range[i])
2372 continue;
2373 pr_devel("cxl release irq range 0x%x: offset: 0x%lx limit: %ld\n",
2374 i, irqs->offset[i],
2375 irqs->range[i]);
2376 hwirq = irqs->offset[i] - phb->msi_base;
2377 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq,
2378 irqs->range[i]);
2379 }
2380}
2381EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges);
2382
2383int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
2384 struct pci_dev *dev, int num)
2385{
2386 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2387 struct pnv_phb *phb = hose->private_data;
2388 int i, hwirq, try;
2389
2390 memset(irqs, 0, sizeof(struct cxl_irq_ranges));
2391
2392 /* 0 is reserved for the multiplexed PSL DSI interrupt */
2393 for (i = 1; i < CXL_IRQ_RANGES && num; i++) {
2394 try = num;
2395 while (try) {
2396 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try);
2397 if (hwirq >= 0)
2398 break;
2399 try /= 2;
2400 }
2401 if (!try)
2402 goto fail;
2403
2404 irqs->offset[i] = phb->msi_base + hwirq;
2405 irqs->range[i] = try;
2406 pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx limit: %li\n",
2407 i, irqs->offset[i], irqs->range[i]);
2408 num -= try;
2409 }
2410 if (num)
2411 goto fail;
2412
2413 return 0;
2414fail:
2415 pnv_cxl_release_hwirq_ranges(irqs, dev);
2416 return -ENOSPC;
2417}
2418EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges);
2419
2420int pnv_cxl_get_irq_count(struct pci_dev *dev)
2421{
2422 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2423 struct pnv_phb *phb = hose->private_data;
2424
2425 return phb->msi_bmp.irq_count;
2426}
2427EXPORT_SYMBOL(pnv_cxl_get_irq_count);
2428
2429int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq,
2430 unsigned int virq)
2431{
2432 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2433 struct pnv_phb *phb = hose->private_data;
2434 unsigned int xive_num = hwirq - phb->msi_base;
2435 struct pnv_ioda_pe *pe;
2436 int rc;
2437
2438 if (!(pe = pnv_ioda_get_pe(dev)))
2439 return -ENODEV;
2440
2441 /* Assign XIVE to PE */
2442 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2443 if (rc) {
2444 pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x "
2445 "hwirq 0x%x XIVE 0x%x PE\n",
2446 pci_name(dev), rc, phb->msi_base, hwirq, xive_num);
2447 return -EIO;
2448 }
2449 set_msi_irq_chip(phb, virq);
2450
2451 return 0;
2452}
2453EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup);
2454#endif
2455
184cd4a3 2456static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
137436c9
GS		 2457 unsigned int hwirq, unsigned int virq,
2458 unsigned int is_64, struct msi_msg *msg)
184cd4a3
BH		 2459{
2460 struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2461 unsigned int xive_num = hwirq - phb->msi_base;
3a1a4661 2462 __be32 data;
184cd4a3
BH		 2463 int rc;
2464
2465 /* No PE assigned ? bail out ... no MSI for you ! */
2466 if (pe == NULL)
2467 return -ENXIO;
2468
2469 /* Check if we have an MVE */
2470 if (pe->mve_number < 0)
2471 return -ENXIO;
2472
b72c1f65 2473 /* Force 32-bit MSI on some broken devices */
36074381 2474 if (dev->no_64bit_msi)
b72c1f65
BH		 2475 is_64 = 0;
2476
184cd4a3
BH		 2477 /* Assign XIVE to PE */
2478 rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2479 if (rc) {
2480 pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2481 pci_name(dev), rc, xive_num);
2482 return -EIO;
2483 }
2484
2485 if (is_64) {
3a1a4661
BH		 2486 __be64 addr64;
2487
184cd4a3
BH		 2488 rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2489 &addr64, &data);
2490 if (rc) {
2491 pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2492 pci_name(dev), rc);
2493 return -EIO;
2494 }
3a1a4661
BH		 2495 msg->address_hi = be64_to_cpu(addr64) >> 32;
2496 msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
184cd4a3 2497 } else {
3a1a4661
BH		 2498 __be32 addr32;
2499
184cd4a3
BH		 2500 rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2501 &addr32, &data);
2502 if (rc) {
2503 pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2504 pci_name(dev), rc);
2505 return -EIO;
2506 }
2507 msg->address_hi = 0;
3a1a4661 2508 msg->address_lo = be32_to_cpu(addr32);
184cd4a3 2509 }
3a1a4661 2510 msg->data = be32_to_cpu(data);
184cd4a3 2511
fd9a1c26 2512 set_msi_irq_chip(phb, virq);
137436c9 2513
184cd4a3
BH		 2514 pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
2515 " address=%x_%08x data=%x PE# %d\n",
2516 pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
2517 msg->address_hi, msg->address_lo, data, pe->pe_number);
2518
2519 return 0;
2520}
2521
2522static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2523{
fb1b55d6 2524 unsigned int count;
184cd4a3
BH		 2525 const __be32 *prop = of_get_property(phb->hose->dn,
2526 "ibm,opal-msi-ranges", NULL);
2527 if (!prop) {
2528 /* BML Fallback */
2529 prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2530 }
2531 if (!prop)
2532 return;
2533
2534 phb->msi_base = be32_to_cpup(prop);
fb1b55d6
GS		 2535 count = be32_to_cpup(prop + 1);
2536 if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
184cd4a3
BH		 2537 pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2538 phb->hose->global_number);
2539 return;
2540 }
fb1b55d6 2541
184cd4a3
BH		 2542 phb->msi_setup = pnv_pci_ioda_msi_setup;
2543 phb->msi32_support = 1;
2544 pr_info(" Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
fb1b55d6 2545 count, phb->msi_base);
184cd4a3
BH		 2546}
2547#else
2548static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
2549#endif /* CONFIG_PCI_MSI */
2550
6e628c7d
WY		 2551#ifdef CONFIG_PCI_IOV
2552static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
2553{
2554 struct pci_controller *hose;
2555 struct pnv_phb *phb;
2556 struct resource *res;
2557 int i;
2558 resource_size_t size;
2559 struct pci_dn *pdn;
5b88ec22 2560 int mul, total_vfs;
6e628c7d
WY		 2561
2562 if (!pdev->is_physfn || pdev->is_added)
2563 return;
2564
2565 hose = pci_bus_to_host(pdev->bus);
2566 phb = hose->private_data;
2567
2568 pdn = pci_get_pdn(pdev);
2569 pdn->vfs_expanded = 0;
2570
5b88ec22
WY		 2571 total_vfs = pci_sriov_get_totalvfs(pdev);
2572 pdn->m64_per_iov = 1;
2573 mul = phb->ioda.total_pe;
2574
2575 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2576 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2577 if (!res->flags || res->parent)
2578 continue;
2579 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2580 dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n",
2581 i, res);
2582 continue;
2583 }
2584
2585 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
2586
2587 /* bigger than 64M */
2588 if (size > (1 << 26)) {
2589 dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n",
2590 i, res);
2591 pdn->m64_per_iov = M64_PER_IOV;
2592 mul = roundup_pow_of_two(total_vfs);
2593 break;
2594 }
2595 }
2596
6e628c7d
WY		 2597 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
2598 res = &pdev->resource[i + PCI_IOV_RESOURCES];
2599 if (!res->flags || res->parent)
2600 continue;
2601 if (!pnv_pci_is_mem_pref_64(res->flags)) {
2602 dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n",
2603 i, res);
2604 continue;
2605 }
2606
2607 dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
2608 size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
5b88ec22 2609 res->end = res->start + size * mul - 1;
6e628c7d
WY		 2610 dev_dbg(&pdev->dev, " %pR\n", res);
2611 dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
5b88ec22 2612 i, res, mul);
6e628c7d 2613 }
5b88ec22 2614 pdn->vfs_expanded = mul;
6e628c7d
WY		 2615}
2616#endif /* CONFIG_PCI_IOV */
2617
11685bec
GS		 2618/*
2619 * This function is supposed to be called on basis of PE from top
2620 * to bottom style. So the the I/O or MMIO segment assigned to
2621 * parent PE could be overrided by its child PEs if necessary.
2622 */
cad5cef6
GKH		 2623static void pnv_ioda_setup_pe_seg(struct pci_controller *hose,
2624 struct pnv_ioda_pe *pe)
11685bec
GS		 2625{
2626 struct pnv_phb *phb = hose->private_data;
2627 struct pci_bus_region region;
2628 struct resource *res;
2629 int i, index;
2630 int rc;
2631
2632 /*
2633 * NOTE: We only care PCI bus based PE for now. For PCI
2634 * device based PE, for example SRIOV sensitive VF should
2635 * be figured out later.
2636 */
2637 BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2638
2639 pci_bus_for_each_resource(pe->pbus, res, i) {
2640 if (!res || !res->flags ||
2641 res->start > res->end)
2642 continue;
2643
2644 if (res->flags & IORESOURCE_IO) {
2645 region.start = res->start - phb->ioda.io_pci_base;
2646 region.end = res->end - phb->ioda.io_pci_base;
2647 index = region.start / phb->ioda.io_segsize;
2648
2649 while (index < phb->ioda.total_pe &&
2650 region.start <= region.end) {
2651 phb->ioda.io_segmap[index] = pe->pe_number;
2652 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2653 pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2654 if (rc != OPAL_SUCCESS) {
2655 pr_err("%s: OPAL error %d when mapping IO "
2656 "segment #%d to PE#%d\n",
2657 __func__, rc, index, pe->pe_number);
2658 break;
2659 }
2660
2661 region.start += phb->ioda.io_segsize;
2662 index++;
2663 }
027fa02f
GS		 2664 } else if ((res->flags & IORESOURCE_MEM) &&
2665 !pnv_pci_is_mem_pref_64(res->flags)) {
11685bec 2666 region.start = res->start -
3fd47f06 2667 hose->mem_offset[0] -
11685bec
GS		 2668 phb->ioda.m32_pci_base;
2669 region.end = res->end -
3fd47f06 2670 hose->mem_offset[0] -
11685bec
GS		 2671 phb->ioda.m32_pci_base;
2672 index = region.start / phb->ioda.m32_segsize;
2673
2674 while (index < phb->ioda.total_pe &&
2675 region.start <= region.end) {
2676 phb->ioda.m32_segmap[index] = pe->pe_number;
2677 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2678 pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2679 if (rc != OPAL_SUCCESS) {
2680 pr_err("%s: OPAL error %d when mapping M32 "
2681 "segment#%d to PE#%d",
2682 __func__, rc, index, pe->pe_number);
2683 break;
2684 }
2685
2686 region.start += phb->ioda.m32_segsize;
2687 index++;
2688 }
2689 }
2690 }
2691}
2692
cad5cef6 2693static void pnv_pci_ioda_setup_seg(void)
11685bec
GS		 2694{
2695 struct pci_controller *tmp, *hose;
2696 struct pnv_phb *phb;
2697 struct pnv_ioda_pe *pe;
2698
2699 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2700 phb = hose->private_data;
2701 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2702 pnv_ioda_setup_pe_seg(hose, pe);
2703 }
2704 }
2705}
2706
cad5cef6 2707static void pnv_pci_ioda_setup_DMA(void)
13395c48
GS		 2708{
2709 struct pci_controller *hose, *tmp;
db1266c8 2710 struct pnv_phb *phb;
13395c48
GS		 2711
2712 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2713 pnv_ioda_setup_dma(hose->private_data);
db1266c8
GS		 2714
2715 /* Mark the PHB initialization done */
2716 phb = hose->private_data;
2717 phb->initialized = 1;
13395c48
GS		 2718 }
2719}
2720
37c367f2
GS		 2721static void pnv_pci_ioda_create_dbgfs(void)
2722{
2723#ifdef CONFIG_DEBUG_FS
2724 struct pci_controller *hose, *tmp;
2725 struct pnv_phb *phb;
2726 char name[16];
2727
2728 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2729 phb = hose->private_data;
2730
2731 sprintf(name, "PCI%04x", hose->global_number);
2732 phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
2733 if (!phb->dbgfs)
2734 pr_warning("%s: Error on creating debugfs on PHB#%x\n",
2735 __func__, hose->global_number);
2736 }
2737#endif /* CONFIG_DEBUG_FS */
2738}
2739
cad5cef6 2740static void pnv_pci_ioda_fixup(void)
fb446ad0
GS		 2741{
2742 pnv_pci_ioda_setup_PEs();
11685bec 2743 pnv_pci_ioda_setup_seg();
13395c48 2744 pnv_pci_ioda_setup_DMA();
e9cc17d4 2745
37c367f2
GS		 2746 pnv_pci_ioda_create_dbgfs();
2747
e9cc17d4 2748#ifdef CONFIG_EEH
e9cc17d4 2749 eeh_init();
dadcd6d6 2750 eeh_addr_cache_build();
e9cc17d4 2751#endif
fb446ad0
GS		 2752}
2753
271fd03a
GS		 2754/*
2755 * Returns the alignment for I/O or memory windows for P2P
2756 * bridges. That actually depends on how PEs are segmented.
2757 * For now, we return I/O or M32 segment size for PE sensitive
2758 * P2P bridges. Otherwise, the default values (4KiB for I/O,
2759 * 1MiB for memory) will be returned.
2760 *
2761 * The current PCI bus might be put into one PE, which was
2762 * create against the parent PCI bridge. For that case, we
2763 * needn't enlarge the alignment so that we can save some
2764 * resources.
2765 */
2766static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2767 unsigned long type)
2768{
2769 struct pci_dev *bridge;
2770 struct pci_controller *hose = pci_bus_to_host(bus);
2771 struct pnv_phb *phb = hose->private_data;
2772 int num_pci_bridges = 0;
2773
2774 bridge = bus->self;
2775 while (bridge) {
2776 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2777 num_pci_bridges++;
2778 if (num_pci_bridges >= 2)
2779 return 1;
2780 }
2781
2782 bridge = bridge->bus->self;
2783 }
2784
262af557
GC		 2785 /* We fail back to M32 if M64 isn't supported */
2786 if (phb->ioda.m64_segsize &&
2787 pnv_pci_is_mem_pref_64(type))
2788 return phb->ioda.m64_segsize;
271fd03a
GS		 2789 if (type & IORESOURCE_MEM)
2790 return phb->ioda.m32_segsize;
2791
2792 return phb->ioda.io_segsize;
2793}
2794
5350ab3f
WY		 2795#ifdef CONFIG_PCI_IOV
2796static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
2797 int resno)
2798{
2799 struct pci_dn *pdn = pci_get_pdn(pdev);
2800 resource_size_t align, iov_align;
2801
2802 iov_align = resource_size(&pdev->resource[resno]);
2803 if (iov_align)
2804 return iov_align;
2805
2806 align = pci_iov_resource_size(pdev, resno);
2807 if (pdn->vfs_expanded)
2808 return pdn->vfs_expanded * align;
2809
2810 return align;
2811}
2812#endif /* CONFIG_PCI_IOV */
2813
184cd4a3
BH		 2814/* Prevent enabling devices for which we couldn't properly
2815 * assign a PE
2816 */
c88c2a18 2817static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
184cd4a3 2818{
db1266c8
GS		 2819 struct pci_controller *hose = pci_bus_to_host(dev->bus);
2820 struct pnv_phb *phb = hose->private_data;
2821 struct pci_dn *pdn;
184cd4a3 2822
db1266c8
GS		 2823 /* The function is probably called while the PEs have
2824 * not be created yet. For example, resource reassignment
2825 * during PCI probe period. We just skip the check if
2826 * PEs isn't ready.
2827 */
2828 if (!phb->initialized)
c88c2a18 2829 return true;
db1266c8 2830
b72c1f65 2831 pdn = pci_get_pdn(dev);
184cd4a3 2832 if (!pdn || pdn->pe_number == IODA_INVALID_PE)
c88c2a18 2833 return false;
db1266c8 2834
c88c2a18 2835 return true;
184cd4a3
BH		 2836}
2837
2838static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
2839 u32 devfn)
2840{
2841 return phb->ioda.pe_rmap[(bus->number << 8) | devfn];
2842}
2843
7a8e6bbf 2844static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
73ed148a 2845{
7a8e6bbf
MN		 2846 struct pnv_phb *phb = hose->private_data;
2847
d1a85eee 2848 opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
73ed148a
BH		 2849 OPAL_ASSERT_RESET);
2850}
2851
92ae0353
DA		 2852static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
2853 .dma_dev_setup = pnv_pci_dma_dev_setup,
2854#ifdef CONFIG_PCI_MSI
2855 .setup_msi_irqs = pnv_setup_msi_irqs,
2856 .teardown_msi_irqs = pnv_teardown_msi_irqs,
2857#endif
2858 .enable_device_hook = pnv_pci_enable_device_hook,
2859 .window_alignment = pnv_pci_window_alignment,
2860 .reset_secondary_bus = pnv_pci_reset_secondary_bus,
763d2d8d 2861 .dma_set_mask = pnv_pci_ioda_dma_set_mask,
7a8e6bbf 2862 .shutdown = pnv_pci_ioda_shutdown,
92ae0353
DA		 2863};
2864
e51df2c1
AB		 2865static void __init pnv_pci_init_ioda_phb(struct device_node *np,
2866 u64 hub_id, int ioda_type)
184cd4a3
BH		 2867{
2868 struct pci_controller *hose;
184cd4a3 2869 struct pnv_phb *phb;
8184616f 2870 unsigned long size, m32map_off, pemap_off, iomap_off = 0;
c681b93c 2871 const __be64 *prop64;
3a1a4661 2872 const __be32 *prop32;
f1b7cc3e 2873 int len;
184cd4a3
BH		 2874 u64 phb_id;
2875 void *aux;
2876 long rc;
2877
58d714ec 2878 pr_info("Initializing IODA%d OPAL PHB %s\n", ioda_type, np->full_name);
184cd4a3
BH		 2879
2880 prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
2881 if (!prop64) {
2882 pr_err(" Missing \"ibm,opal-phbid\" property !\n");
2883 return;
2884 }
2885 phb_id = be64_to_cpup(prop64);
2886 pr_debug(" PHB-ID : 0x%016llx\n", phb_id);
2887
e39f223f 2888 phb = memblock_virt_alloc(sizeof(struct pnv_phb), 0);
58d714ec
GS		 2889
2890 /* Allocate PCI controller */
58d714ec
GS		 2891 phb->hose = hose = pcibios_alloc_controller(np);
2892 if (!phb->hose) {
2893 pr_err(" Can't allocate PCI controller for %s\n",
184cd4a3 2894 np->full_name);
e39f223f 2895 memblock_free(__pa(phb), sizeof(struct pnv_phb));
184cd4a3
BH		 2896 return;
2897 }
2898
2899 spin_lock_init(&phb->lock);
f1b7cc3e
GS		 2900 prop32 = of_get_property(np, "bus-range", &len);
2901 if (prop32 && len == 8) {
3a1a4661
BH		 2902 hose->first_busno = be32_to_cpu(prop32[0]);
2903 hose->last_busno = be32_to_cpu(prop32[1]);
f1b7cc3e
GS		 2904 } else {
2905 pr_warn(" Broken <bus-range> on %s\n", np->full_name);
2906 hose->first_busno = 0;
2907 hose->last_busno = 0xff;
2908 }
184cd4a3 2909 hose->private_data = phb;
e9cc17d4 2910 phb->hub_id = hub_id;
184cd4a3 2911 phb->opal_id = phb_id;
aa0c033f 2912 phb->type = ioda_type;
781a868f 2913 mutex_init(&phb->ioda.pe_alloc_mutex);
184cd4a3 2914
cee72d5b
BH		 2915 /* Detect specific models for error handling */
2916 if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
2917 phb->model = PNV_PHB_MODEL_P7IOC;
f3d40c25 2918 else if (of_device_is_compatible(np, "ibm,power8-pciex"))
aa0c033f 2919 phb->model = PNV_PHB_MODEL_PHB3;
cee72d5b
BH		 2920 else
2921 phb->model = PNV_PHB_MODEL_UNKNOWN;
2922
aa0c033f 2923 /* Parse 32-bit and IO ranges (if any) */
2f1ec02e 2924 pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
184cd4a3 2925
aa0c033f 2926 /* Get registers */
184cd4a3
BH		 2927 phb->regs = of_iomap(np, 0);
2928 if (phb->regs == NULL)
2929 pr_err(" Failed to map registers !\n");
2930
184cd4a3 2931 /* Initialize more IODA stuff */
36954dc7 2932 phb->ioda.total_pe = 1;
aa0c033f 2933 prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
36954dc7 2934 if (prop32)
3a1a4661 2935 phb->ioda.total_pe = be32_to_cpup(prop32);
36954dc7
GS		 2936 prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
2937 if (prop32)
2938 phb->ioda.reserved_pe = be32_to_cpup(prop32);
262af557
GC		 2939
2940 /* Parse 64-bit MMIO range */
2941 pnv_ioda_parse_m64_window(phb);
2942
184cd4a3 2943 phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
aa0c033f 2944 /* FW Has already off top 64k of M32 space (MSI space) */
184cd4a3
BH		 2945 phb->ioda.m32_size += 0x10000;
2946
2947 phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe;
3fd47f06 2948 phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
184cd4a3
BH		 2949 phb->ioda.io_size = hose->pci_io_size;
2950 phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe;
2951 phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
2952
c35d2a8c 2953 /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
184cd4a3
BH		 2954 size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long));
2955 m32map_off = size;
e47747f4 2956 size += phb->ioda.total_pe * sizeof(phb->ioda.m32_segmap[0]);
c35d2a8c
GS		 2957 if (phb->type == PNV_PHB_IODA1) {
2958 iomap_off = size;
2959 size += phb->ioda.total_pe * sizeof(phb->ioda.io_segmap[0]);
2960 }
184cd4a3
BH		 2961 pemap_off = size;
2962 size += phb->ioda.total_pe * sizeof(struct pnv_ioda_pe);
e39f223f 2963 aux = memblock_virt_alloc(size, 0);
184cd4a3
BH		 2964 phb->ioda.pe_alloc = aux;
2965 phb->ioda.m32_segmap = aux + m32map_off;
c35d2a8c
GS		 2966 if (phb->type == PNV_PHB_IODA1)
2967 phb->ioda.io_segmap = aux + iomap_off;
184cd4a3 2968 phb->ioda.pe_array = aux + pemap_off;
36954dc7 2969 set_bit(phb->ioda.reserved_pe, phb->ioda.pe_alloc);
184cd4a3 2970
7ebdf956 2971 INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
184cd4a3 2972 INIT_LIST_HEAD(&phb->ioda.pe_list);
781a868f 2973 mutex_init(&phb->ioda.pe_list_mutex);
184cd4a3
BH		 2974
2975 /* Calculate how many 32-bit TCE segments we have */
2976 phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
2977
aa0c033f 2978#if 0 /* We should really do that ... */
184cd4a3
BH		 2979 rc = opal_pci_set_phb_mem_window(opal->phb_id,
2980 window_type,
2981 window_num,
2982 starting_real_address,
2983 starting_pci_address,
2984 segment_size);
2985#endif
2986
262af557
GC		 2987 pr_info(" %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
2988 phb->ioda.total_pe, phb->ioda.reserved_pe,
2989 phb->ioda.m32_size, phb->ioda.m32_segsize);
2990 if (phb->ioda.m64_size)
2991 pr_info(" M64: 0x%lx [segment=0x%lx]\n",
2992 phb->ioda.m64_size, phb->ioda.m64_segsize);
2993 if (phb->ioda.io_size)
2994 pr_info(" IO: 0x%x [segment=0x%x]\n",
2995 phb->ioda.io_size, phb->ioda.io_segsize);
2996
184cd4a3 2997
184cd4a3 2998 phb->hose->ops = &pnv_pci_ops;
49dec922
GS		 2999 phb->get_pe_state = pnv_ioda_get_pe_state;
3000 phb->freeze_pe = pnv_ioda_freeze_pe;
3001 phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
184cd4a3
BH		 3002
3003 /* Setup RID -> PE mapping function */
3004 phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe;
3005
3006 /* Setup TCEs */
3007 phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
fe7e85c6 3008 phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask;
184cd4a3
BH		 3009
3010 /* Setup MSI support */
3011 pnv_pci_init_ioda_msis(phb);
3012
c40a4210
GS		 3013 /*
3014 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
3015 * to let the PCI core do resource assignment. It's supposed
3016 * that the PCI core will do correct I/O and MMIO alignment
3017 * for the P2P bridge bars so that each PCI bus (excluding
3018 * the child P2P bridges) can form individual PE.
184cd4a3 3019 */
fb446ad0 3020 ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
92ae0353 3021 hose->controller_ops = pnv_pci_ioda_controller_ops;
ad30cb99 3022
6e628c7d
WY		 3023#ifdef CONFIG_PCI_IOV
3024 ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
5350ab3f 3025 ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
ad30cb99
ME		 3026#endif
3027
c40a4210 3028 pci_add_flags(PCI_REASSIGN_ALL_RSRC);
184cd4a3
BH		 3029
3030 /* Reset IODA tables to a clean state */
d1a85eee 3031 rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
184cd4a3 3032 if (rc)
f11fe552 3033 pr_warning(" OPAL Error %ld performing IODA table reset !\n", rc);
361f2a2a
GS		 3034
3035 /* If we're running in kdump kerenl, the previous kerenl never
3036 * shutdown PCI devices correctly. We already got IODA table
3037 * cleaned out. So we have to issue PHB reset to stop all PCI
3038 * transactions from previous kerenl.
3039 */
3040 if (is_kdump_kernel()) {
3041 pr_info(" Issue PHB reset ...\n");
cadf364d
GS		 3042 pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
3043 pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
361f2a2a 3044 }
262af557 3045
9e9e8935
GS		 3046 /* Remove M64 resource if we can't configure it successfully */
3047 if (!phb->init_m64 || phb->init_m64(phb))
262af557 3048 hose->mem_resources[1].flags = 0;
aa0c033f
GS		 3049}
3050
67975005 3051void __init pnv_pci_init_ioda2_phb(struct device_node *np)
aa0c033f 3052{
e9cc17d4 3053 pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
184cd4a3
BH		 3054}
3055
3056void __init pnv_pci_init_ioda_hub(struct device_node *np)
3057{
3058 struct device_node *phbn;
c681b93c 3059 const __be64 *prop64;
184cd4a3
BH		 3060 u64 hub_id;
3061
3062 pr_info("Probing IODA IO-Hub %s\n", np->full_name);
3063
3064 prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
3065 if (!prop64) {
3066 pr_err(" Missing \"ibm,opal-hubid\" property !\n");
3067 return;
3068 }
3069 hub_id = be64_to_cpup(prop64);
3070 pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
3071
3072 /* Count child PHBs */
3073 for_each_child_of_node(np, phbn) {
3074 /* Look for IODA1 PHBs */
3075 if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
e9cc17d4 3076 pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
184cd4a3
BH		 3077 }
3078}
Linux kernel - Deliverables
This page took 0.307413 seconds and 5 git commands to generate.