[deliverable/linux.git] / arch / powerpc / platforms / powernv / pci.c

/*
 * Support PCI/PCIe on PowerNV platforms
 *
 * Currently supports only P5IOC2
 *
 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/irq.h>
#include <linux/io.h>
#include <linux/msi.h>

#include <asm/sections.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
#include <asm/ppc-pci.h>
#include <asm/opal.h>
#include <asm/iommu.h>
#include <asm/tce.h>
#include <asm/abs_addr.h>

#include "powernv.h"
#include "pci.h"

/* Delay in usec */
#define PCI_RESET_DELAY_US	3000000

#define cfg_dbg(fmt...)	do { } while(0)
//#define cfg_dbg(fmt...)	printk(fmt)

#ifdef CONFIG_PCI_MSI
static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	return (phb && phb->msi_map) ? 0 : -ENODEV;
}

static unsigned int pnv_get_one_msi(struct pnv_phb *phb)
{
	unsigned int id;

	spin_lock(&phb->lock);
	id = find_next_zero_bit(phb->msi_map, phb->msi_count, phb->msi_next);
	if (id >= phb->msi_count && phb->msi_next)
		id = find_next_zero_bit(phb->msi_map, phb->msi_count, 0);
	if (id >= phb->msi_count) {
		spin_unlock(&phb->lock);
		return 0;
	}
	__set_bit(id, phb->msi_map);
	spin_unlock(&phb->lock);
	return id + phb->msi_base;
}

static void pnv_put_msi(struct pnv_phb *phb, unsigned int hwirq)
{
	unsigned int id;

	if (WARN_ON(hwirq < phb->msi_base ||
		    hwirq >= (phb->msi_base + phb->msi_count)))
		return;
	id = hwirq - phb->msi_base;
	spin_lock(&phb->lock);
	__clear_bit(id, phb->msi_map);
	spin_unlock(&phb->lock);
}

static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;
	struct msi_msg msg;
	unsigned int hwirq, virq;
	int rc;

	if (WARN_ON(!phb))
		return -ENODEV;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
			pr_warn("%s: Supports only 64-bit MSIs\n",
				pci_name(pdev));
			return -ENXIO;
		}
		hwirq = pnv_get_one_msi(phb);
		if (!hwirq) {
			pr_warn("%s: Failed to find a free MSI\n",
				pci_name(pdev));
			return -ENOSPC;
		}
		virq = irq_create_mapping(NULL, hwirq);
		if (virq == NO_IRQ) {
			pr_warn("%s: Failed to map MSI to linux irq\n",
				pci_name(pdev));
			pnv_put_msi(phb, hwirq);
			return -ENOMEM;
		}
		rc = phb->msi_setup(phb, pdev, hwirq, entry->msi_attrib.is_64,
				    &msg);
		if (rc) {
			pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
			irq_dispose_mapping(virq);
			pnv_put_msi(phb, hwirq);
			return rc;
		}
		irq_set_msi_desc(virq, entry);
		write_msi_msg(virq, &msg);
	}
	return 0;
}

static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;

	if (WARN_ON(!phb))
		return;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (entry->irq == NO_IRQ)
			continue;
		irq_set_msi_desc(entry->irq, NULL);
		pnv_put_msi(phb, virq_to_hw(entry->irq));
		irq_dispose_mapping(entry->irq);
	}
}
#endif /* CONFIG_PCI_MSI */

static void pnv_pci_config_check_eeh(struct pnv_phb *phb, struct pci_bus *bus,
				     u32 bdfn)
{
	s64	rc;
	u8	fstate;
	u16	pcierr;
	u32	pe_no;

	/* Get PE# if we support IODA */
	pe_no = phb->bdfn_to_pe ? phb->bdfn_to_pe(phb, bus, bdfn & 0xff) : 0;

	/* Read freeze status */
	rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no, &fstate, &pcierr,
					NULL);
	if (rc) {
		pr_warning("PCI %d: Failed to read EEH status for PE#%d,"
			   " err %lld\n", phb->hose->global_number, pe_no, rc);
		return;
	}
	cfg_dbg(" -> EEH check, bdfn=%04x PE%d fstate=%x\n",
		bdfn, pe_no, fstate);
	if (fstate != 0) {
		rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
					      OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
		if (rc) {
			pr_warning("PCI %d: Failed to clear EEH freeze state"
				   " for PE#%d, err %lld\n",
				   phb->hose->global_number, pe_no, rc);
		}
	}
}

static int pnv_pci_read_config(struct pci_bus *bus,
			       unsigned int devfn,
			       int where, int size, u32 *val)
{
	struct pci_controller *hose = pci_bus_to_host(bus);
	struct pnv_phb *phb = hose->private_data;
	u32 bdfn = (((uint64_t)bus->number) << 8) | devfn;
	s64 rc;

	if (hose == NULL)
		return PCIBIOS_DEVICE_NOT_FOUND;

	switch (size) {
	case 1: {
		u8 v8;
		rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
		*val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
		break;
	}
	case 2: {
		u16 v16;
		rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
						   &v16);
		*val = (rc == OPAL_SUCCESS) ? v16 : 0xffff;
		break;
	}
	case 4: {
		u32 v32;
		rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
		*val = (rc == OPAL_SUCCESS) ? v32 : 0xffffffff;
		break;
	}
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
	cfg_dbg("pnv_pci_read_config bus: %x devfn: %x +%x/%x -> %08x\n",
		bus->number, devfn, where, size, *val);

	/* Check if the PHB got frozen due to an error (no response) */
	pnv_pci_config_check_eeh(phb, bus, bdfn);

	return PCIBIOS_SUCCESSFUL;
}

static int pnv_pci_write_config(struct pci_bus *bus,
				unsigned int devfn,
				int where, int size, u32 val)
{
	struct pci_controller *hose = pci_bus_to_host(bus);
	struct pnv_phb *phb = hose->private_data;
	u32 bdfn = (((uint64_t)bus->number) << 8) | devfn;

	if (hose == NULL)
		return PCIBIOS_DEVICE_NOT_FOUND;

	cfg_dbg("pnv_pci_write_config bus: %x devfn: %x +%x/%x -> %08x\n",
		bus->number, devfn, where, size, val);
	switch (size) {
	case 1:
		opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
		break;
	case 2:
		opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
		break;
	case 4:
		opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
		break;
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
	/* Check if the PHB got frozen due to an error (no response) */
	pnv_pci_config_check_eeh(phb, bus, bdfn);

	return PCIBIOS_SUCCESSFUL;
}

struct pci_ops pnv_pci_ops = {
	.read = pnv_pci_read_config,
	.write = pnv_pci_write_config,
};


static void pnv_tce_invalidate(struct iommu_table *tbl,
			       u64 *startp, u64 *endp)
{
	u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index;
	unsigned long start, end, inc;

	start = __pa(startp);
	end = __pa(endp);


	/* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
	if (tbl->it_busno) {
		start <<= 12;
		end <<= 12;
		inc = 128 << 12;
		start |= tbl->it_busno;
		end |= tbl->it_busno;
	}
	/* p7ioc-style invalidation, 2 TCEs per write */
	else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
		start |= (1ull << 63);
		end |= (1ull << 63);
		inc = 16;
	}
	/* Default (older HW) */
	else
		inc = 128;

	end |= inc - 1;		/* round up end to be different than start */

	mb(); /* Ensure above stores are visible */
	while (start <= end) {
		__raw_writeq(start, invalidate);
		start += inc;
	}
	/* The iommu layer will do another mb() for us on build() and
	 * we don't care on free()
	 */
}


static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
			 unsigned long uaddr, enum dma_data_direction direction,
			 struct dma_attrs *attrs)
{
	u64 proto_tce;
	u64 *tcep, *tces;
	u64 rpn;

	proto_tce = TCE_PCI_READ; // Read allowed

	if (direction != DMA_TO_DEVICE)
		proto_tce |= TCE_PCI_WRITE;

	tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset;
	rpn = __pa(uaddr) >> TCE_SHIFT;

	while (npages--)
		*(tcep++) = proto_tce | (rpn++ << TCE_RPN_SHIFT);

	/* Some implementations won't cache invalid TCEs and thus may not
	 * need that flush. We'll probably turn it_type into a bit mask
	 * of flags if that becomes the case
	 */
	if (tbl->it_type & TCE_PCI_SWINV_CREATE)
		pnv_tce_invalidate(tbl, tces, tcep - 1);

	return 0;
}

static void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
{
	u64 *tcep, *tces;

	tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset;

	while (npages--)
		*(tcep++) = 0;

	if (tbl->it_type & TCE_PCI_SWINV_FREE)
		pnv_tce_invalidate(tbl, tces, tcep - 1);
}

void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
			       void *tce_mem, u64 tce_size,
			       u64 dma_offset)
{
	tbl->it_blocksize = 16;
	tbl->it_base = (unsigned long)tce_mem;
	tbl->it_offset = dma_offset >> IOMMU_PAGE_SHIFT;
	tbl->it_index = 0;
	tbl->it_size = tce_size >> 3;
	tbl->it_busno = 0;
	tbl->it_type = TCE_PCI;
}

static struct iommu_table * __devinit
pnv_pci_setup_bml_iommu(struct pci_controller *hose)
{
	struct iommu_table *tbl;
	const __be64 *basep, *swinvp;
	const __be32 *sizep;

	basep = of_get_property(hose->dn, "linux,tce-base", NULL);
	sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
	if (basep == NULL || sizep == NULL) {
		pr_err("PCI: %s has missing tce entries !\n",
		       hose->dn->full_name);
		return NULL;
	}
	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
	if (WARN_ON(!tbl))
		return NULL;
	pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
				  be32_to_cpup(sizep), 0);
	iommu_init_table(tbl, hose->node);

	/* Deal with SW invalidated TCEs when needed (BML way) */
	swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
				 NULL);
	if (swinvp) {
		tbl->it_busno = swinvp[1];
		tbl->it_index = (unsigned long)ioremap(swinvp[0], 8);
		tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
	}
	return tbl;
}

static void __devinit pnv_pci_dma_fallback_setup(struct pci_controller *hose,
						 struct pci_dev *pdev)
{
	struct device_node *np = pci_bus_to_OF_node(hose->bus);
	struct pci_dn *pdn;

	if (np == NULL)
		return;
	pdn = PCI_DN(np);
	if (!pdn->iommu_table)
		pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
	if (!pdn->iommu_table)
		return;
	set_iommu_table_base(&pdev->dev, pdn->iommu_table);
}

static void __devinit pnv_pci_dma_dev_setup(struct pci_dev *pdev)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	/* If we have no phb structure, try to setup a fallback based on
	 * the device-tree (RTAS PCI for example)
	 */
	if (phb && phb->dma_dev_setup)
		phb->dma_dev_setup(phb, pdev);
	else
		pnv_pci_dma_fallback_setup(hose, pdev);
}

/* Fixup wrong class code in p7ioc root complex */
static void __devinit pnv_p7ioc_rc_quirk(struct pci_dev *dev)
{
	dev->class = PCI_CLASS_BRIDGE_PCI << 8;
}
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);

static int pnv_pci_probe_mode(struct pci_bus *bus)
{
	struct pci_controller *hose = pci_bus_to_host(bus);
	const __be64 *tstamp;
	u64 now, target;


	/* We hijack this as a way to ensure we have waited long
	 * enough since the reset was lifted on the PCI bus
	 */
	if (bus != hose->bus)
		return PCI_PROBE_NORMAL;
	tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
	if (!tstamp || !*tstamp)
		return PCI_PROBE_NORMAL;

	now = mftb() / tb_ticks_per_usec;
	target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
		+ PCI_RESET_DELAY_US;

	pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
		 hose->global_number, target, now);

	if (now < target)
		msleep((target - now + 999) / 1000);

	return PCI_PROBE_NORMAL;
}

void __init pnv_pci_init(void)
{
	struct device_node *np;

	pci_set_flags(PCI_CAN_SKIP_ISA_ALIGN);

	/* We do not want to just probe */
	pci_probe_only = 0;

	/* OPAL absent, try POPAL first then RTAS detection of PHBs */
	if (!firmware_has_feature(FW_FEATURE_OPAL)) {
#ifdef CONFIG_PPC_POWERNV_RTAS
		init_pci_config_tokens();
		find_and_init_phbs();
#endif /* CONFIG_PPC_POWERNV_RTAS */
	} else {
		/* OPAL is here, do our normal stuff */

		/* Look for p5ioc2 IO-Hubs */
		for_each_compatible_node(np, NULL, "ibm,p5ioc2")
			pnv_pci_init_p5ioc2_hub(np);
	}

	/* Setup the linkage between OF nodes and PHBs */
	pci_devs_phb_init();

	/* Configure IOMMU DMA hooks */
	ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
	ppc_md.tce_build = pnv_tce_build;
	ppc_md.tce_free = pnv_tce_free;
	ppc_md.pci_probe_mode = pnv_pci_probe_mode;
	set_pci_dma_ops(&dma_iommu_ops);

	/* Configure MSIs */
#ifdef CONFIG_PCI_MSI
	ppc_md.msi_check_device = pnv_msi_check_device;
	ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
	ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
#endif
}
Commit	Line	Data
61305a96 BH	1	/*
	2	* Support PCI/PCIe on PowerNV platforms
	3	*
	4	* Currently supports only P5IOC2
	5	*
	6	* Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
	7	*
	8	* This program is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU General Public License
	10	* as published by the Free Software Foundation; either version
	11	* 2 of the License, or (at your option) any later version.
	12	*/
	13
	14	#include <linux/kernel.h>
	15	#include <linux/pci.h>
	16	#include <linux/delay.h>
	17	#include <linux/string.h>
	18	#include <linux/init.h>
	19	#include <linux/bootmem.h>
	20	#include <linux/irq.h>
	21	#include <linux/io.h>
c1a2562a	22	#include <linux/msi.h>
61305a96 BH	23
	24	#include <asm/sections.h>
	25	#include <asm/io.h>
	26	#include <asm/prom.h>
	27	#include <asm/pci-bridge.h>
	28	#include <asm/machdep.h>
	29	#include <asm/ppc-pci.h>
	30	#include <asm/opal.h>
	31	#include <asm/iommu.h>
	32	#include <asm/tce.h>
	33	#include <asm/abs_addr.h>
	34
	35	#include "powernv.h"
	36	#include "pci.h"
	37
82ba129b BH	38	/* Delay in usec */
82ba129b BH	39	#define PCI_RESET_DELAY_US 3000000
61305a96 BH	40
	41	#define cfg_dbg(fmt...) do { } while(0)
	42	//#define cfg_dbg(fmt...) printk(fmt)
	43
c1a2562a BH	44	#ifdef CONFIG_PCI_MSI
	45	static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type)
	46	{
	47	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	48	struct pnv_phb *phb = hose->private_data;
	49
	50	return (phb && phb->msi_map) ? 0 : -ENODEV;
	51	}
	52
	53	static unsigned int pnv_get_one_msi(struct pnv_phb *phb)
	54	{
	55	unsigned int id;
	56
	57	spin_lock(&phb->lock);
	58	id = find_next_zero_bit(phb->msi_map, phb->msi_count, phb->msi_next);
	59	if (id >= phb->msi_count && phb->msi_next)
	60	id = find_next_zero_bit(phb->msi_map, phb->msi_count, 0);
	61	if (id >= phb->msi_count) {
	62	spin_unlock(&phb->lock);
	63	return 0;
	64	}
	65	__set_bit(id, phb->msi_map);
	66	spin_unlock(&phb->lock);
	67	return id + phb->msi_base;
	68	}
	69
	70	static void pnv_put_msi(struct pnv_phb *phb, unsigned int hwirq)
	71	{
	72	unsigned int id;
	73
	74	if (WARN_ON(hwirq < phb->msi_base \|\|
	75	hwirq >= (phb->msi_base + phb->msi_count)))
	76	return;
	77	id = hwirq - phb->msi_base;
	78	spin_lock(&phb->lock);
	79	__clear_bit(id, phb->msi_map);
	80	spin_unlock(&phb->lock);
	81	}
	82
	83	static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
	84	{
	85	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	86	struct pnv_phb *phb = hose->private_data;
	87	struct msi_desc *entry;
	88	struct msi_msg msg;
	89	unsigned int hwirq, virq;
	90	int rc;
	91
	92	if (WARN_ON(!phb))
	93	return -ENODEV;
	94
	95	list_for_each_entry(entry, &pdev->msi_list, list) {
	96	if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
	97	pr_warn("%s: Supports only 64-bit MSIs\n",
	98	pci_name(pdev));
	99	return -ENXIO;
	100	}
	101	hwirq = pnv_get_one_msi(phb);
	102	if (!hwirq) {
	103	pr_warn("%s: Failed to find a free MSI\n",
	104	pci_name(pdev));
	105	return -ENOSPC;
	106	}
	107	virq = irq_create_mapping(NULL, hwirq);
108	if (virq == NO_IRQ) {
109	pr_warn("%s: Failed to map MSI to linux irq\n",
110	pci_name(pdev));
111	pnv_put_msi(phb, hwirq);
112	return -ENOMEM;
113	}
114	rc = phb->msi_setup(phb, pdev, hwirq, entry->msi_attrib.is_64,
115	&msg);
116	if (rc) {
117	pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
118	irq_dispose_mapping(virq);
119	pnv_put_msi(phb, hwirq);
120	return rc;
121	}
122	irq_set_msi_desc(virq, entry);
123	write_msi_msg(virq, &msg);
124	}
125	return 0;
126	}
127
128	static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
129	{
130	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
131	struct pnv_phb *phb = hose->private_data;
132	struct msi_desc *entry;
133
134	if (WARN_ON(!phb))
135	return;
136
137	list_for_each_entry(entry, &pdev->msi_list, list) {
138	if (entry->irq == NO_IRQ)
139	continue;
140	irq_set_msi_desc(entry->irq, NULL);
141	pnv_put_msi(phb, virq_to_hw(entry->irq));
142	irq_dispose_mapping(entry->irq);
143	}
144	}
145	#endif /* CONFIG_PCI_MSI */
61305a96 BH	146
	147	static void pnv_pci_config_check_eeh(struct pnv_phb phb, struct pci_bus bus,
	148	u32 bdfn)
	149	{
	150	s64 rc;
	151	u8 fstate;
	152	u16 pcierr;
	153	u32 pe_no;
	154
	155	/* Get PE# if we support IODA */
	156	pe_no = phb->bdfn_to_pe ? phb->bdfn_to_pe(phb, bus, bdfn & 0xff) : 0;
	157
	158	/* Read freeze status */
	159	rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no, &fstate, &pcierr,
	160	NULL);
	161	if (rc) {
	162	pr_warning("PCI %d: Failed to read EEH status for PE#%d,"
	163	" err %lld\n", phb->hose->global_number, pe_no, rc);
	164	return;
	165	}
	166	cfg_dbg(" -> EEH check, bdfn=%04x PE%d fstate=%x\n",
	167	bdfn, pe_no, fstate);
	168	if (fstate != 0) {
	169	rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
	170	OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
	171	if (rc) {
	172	pr_warning("PCI %d: Failed to clear EEH freeze state"
	173	" for PE#%d, err %lld\n",
	174	phb->hose->global_number, pe_no, rc);
	175	}
	176	}
	177	}
	178
	179	static int pnv_pci_read_config(struct pci_bus *bus,
	180	unsigned int devfn,
	181	int where, int size, u32 *val)
	182	{
	183	struct pci_controller *hose = pci_bus_to_host(bus);
	184	struct pnv_phb *phb = hose->private_data;
	185	u32 bdfn = (((uint64_t)bus->number) << 8) \| devfn;
	186	s64 rc;
	187
	188	if (hose == NULL)
	189	return PCIBIOS_DEVICE_NOT_FOUND;
	190
	191	switch (size) {
	192	case 1: {
	193	u8 v8;
	194	rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
	195	*val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
	196	break;
	197	}
	198	case 2: {
	199	u16 v16;
	200	rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
	201	&v16);
	202	*val = (rc == OPAL_SUCCESS) ? v16 : 0xffff;
	203	break;
	204	}
	205	case 4: {
	206	u32 v32;
	207	rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
	208	*val = (rc == OPAL_SUCCESS) ? v32 : 0xffffffff;
	209	break;
210	}
211	default:
212	return PCIBIOS_FUNC_NOT_SUPPORTED;
213	}
214	cfg_dbg("pnv_pci_read_config bus: %x devfn: %x +%x/%x -> %08x\n",
215	bus->number, devfn, where, size, *val);
216
217	/* Check if the PHB got frozen due to an error (no response) */
218	pnv_pci_config_check_eeh(phb, bus, bdfn);
219
220	return PCIBIOS_SUCCESSFUL;
221	}
222
223	static int pnv_pci_write_config(struct pci_bus *bus,
224	unsigned int devfn,
225	int where, int size, u32 val)
226	{
227	struct pci_controller *hose = pci_bus_to_host(bus);
228	struct pnv_phb *phb = hose->private_data;
229	u32 bdfn = (((uint64_t)bus->number) << 8) \| devfn;
230
231	if (hose == NULL)
232	return PCIBIOS_DEVICE_NOT_FOUND;
233
234	cfg_dbg("pnv_pci_write_config bus: %x devfn: %x +%x/%x -> %08x\n",
235	bus->number, devfn, where, size, val);
236	switch (size) {
237	case 1:
238	opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
239	break;
240	case 2:
241	opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
242	break;
243	case 4:
244	opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
245	break;
246	default:
247	return PCIBIOS_FUNC_NOT_SUPPORTED;
248	}
249	/* Check if the PHB got frozen due to an error (no response) */
250	pnv_pci_config_check_eeh(phb, bus, bdfn);
251
252	return PCIBIOS_SUCCESSFUL;
253	}
254
255	struct pci_ops pnv_pci_ops = {
256	.read = pnv_pci_read_config,
257	.write = pnv_pci_write_config,
258	};
259
1f1616e8 BH	260
	261	static void pnv_tce_invalidate(struct iommu_table *tbl,
	262	u64 startp, u64 endp)
	263	{
	264	u64 __iomem invalidate = (u64 __iomem )tbl->it_index;
	265	unsigned long start, end, inc;
	266
	267	start = __pa(startp);
	268	end = __pa(endp);
	269
	270
	271	/* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */
	272	if (tbl->it_busno) {
	273	start <<= 12;
	274	end <<= 12;
	275	inc = 128 << 12;
	276	start \|= tbl->it_busno;
	277	end \|= tbl->it_busno;
	278	}
	279	/* p7ioc-style invalidation, 2 TCEs per write */
	280	else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {
	281	start \|= (1ull << 63);
	282	end \|= (1ull << 63);
	283	inc = 16;
	284	}
	285	/* Default (older HW) */
	286	else
	287	inc = 128;
	288
	289	end \|= inc - 1; /* round up end to be different than start */
	290
	291	mb(); /* Ensure above stores are visible */
	292	while (start <= end) {
	293	__raw_writeq(start, invalidate);
	294	start += inc;
	295	}
	296	/* The iommu layer will do another mb() for us on build() and
	297	* we don't care on free()
	298	*/
	299	}
	300
	301
61305a96 BH	302	static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
	303	unsigned long uaddr, enum dma_data_direction direction,
	304	struct dma_attrs *attrs)
	305	{
	306	u64 proto_tce;
1f1616e8	307	u64 tcep, tces;
61305a96 BH	308	u64 rpn;
	309
	310	proto_tce = TCE_PCI_READ; // Read allowed
	311
	312	if (direction != DMA_TO_DEVICE)
	313	proto_tce \|= TCE_PCI_WRITE;
	314
1f1616e8 BH	315	tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset;
1f1616e8 BH	316	rpn = __pa(uaddr) >> TCE_SHIFT;
61305a96	317
1f1616e8 BH	318	while (npages--)
	319	*(tcep++) = proto_tce \| (rpn++ << TCE_RPN_SHIFT);
	320
	321	/* Some implementations won't cache invalid TCEs and thus may not
	322	* need that flush. We'll probably turn it_type into a bit mask
	323	* of flags if that becomes the case
	324	*/
	325	if (tbl->it_type & TCE_PCI_SWINV_CREATE)
	326	pnv_tce_invalidate(tbl, tces, tcep - 1);
61305a96	327
61305a96 BH	328	return 0;
	329	}
	330
	331	static void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
	332	{
1f1616e8 BH	333	u64 tcep, tces;
	334
	335	tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset;
61305a96 BH	336
	337	while (npages--)
	338	*(tcep++) = 0;
1f1616e8 BH	339
	340	if (tbl->it_type & TCE_PCI_SWINV_FREE)
	341	pnv_tce_invalidate(tbl, tces, tcep - 1);
61305a96 BH	342	}
	343
	344	void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
	345	void *tce_mem, u64 tce_size,
	346	u64 dma_offset)
	347	{
	348	tbl->it_blocksize = 16;
	349	tbl->it_base = (unsigned long)tce_mem;
	350	tbl->it_offset = dma_offset >> IOMMU_PAGE_SHIFT;
	351	tbl->it_index = 0;
	352	tbl->it_size = tce_size >> 3;
	353	tbl->it_busno = 0;
	354	tbl->it_type = TCE_PCI;
	355	}
	356
	357	static struct iommu_table * __devinit
	358	pnv_pci_setup_bml_iommu(struct pci_controller *hose)
	359	{
	360	struct iommu_table *tbl;
1f1616e8	361	const __be64 basep, swinvp;
61305a96 BH	362	const __be32 *sizep;
	363
	364	basep = of_get_property(hose->dn, "linux,tce-base", NULL);
	365	sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
	366	if (basep == NULL \|\| sizep == NULL) {
1f1616e8 BH	367	pr_err("PCI: %s has missing tce entries !\n",
1f1616e8 BH	368	hose->dn->full_name);
61305a96 BH	369	return NULL;
	370	}
	371	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
	372	if (WARN_ON(!tbl))
	373	return NULL;
	374	pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
	375	be32_to_cpup(sizep), 0);
	376	iommu_init_table(tbl, hose->node);
1f1616e8 BH	377
	378	/* Deal with SW invalidated TCEs when needed (BML way) */
	379	swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
	380	NULL);
	381	if (swinvp) {
	382	tbl->it_busno = swinvp[1];
	383	tbl->it_index = (unsigned long)ioremap(swinvp[0], 8);
	384	tbl->it_type = TCE_PCI_SWINV_CREATE \| TCE_PCI_SWINV_FREE;
	385	}
61305a96 BH	386	return tbl;
	387	}
	388
	389	static void __devinit pnv_pci_dma_fallback_setup(struct pci_controller *hose,
	390	struct pci_dev *pdev)
	391	{
	392	struct device_node *np = pci_bus_to_OF_node(hose->bus);
	393	struct pci_dn *pdn;
	394
	395	if (np == NULL)
	396	return;
	397	pdn = PCI_DN(np);
	398	if (!pdn->iommu_table)
	399	pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
	400	if (!pdn->iommu_table)
	401	return;
	402	set_iommu_table_base(&pdev->dev, pdn->iommu_table);
	403	}
	404
	405	static void __devinit pnv_pci_dma_dev_setup(struct pci_dev *pdev)
	406	{
	407	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	408	struct pnv_phb *phb = hose->private_data;
	409
	410	/* If we have no phb structure, try to setup a fallback based on
	411	* the device-tree (RTAS PCI for example)
	412	*/
	413	if (phb && phb->dma_dev_setup)
	414	phb->dma_dev_setup(phb, pdev);
	415	else
	416	pnv_pci_dma_fallback_setup(hose, pdev);
	417	}
	418
ca45cfe3 BH	419	/* Fixup wrong class code in p7ioc root complex */
	420	static void __devinit pnv_p7ioc_rc_quirk(struct pci_dev *dev)
	421	{
	422	dev->class = PCI_CLASS_BRIDGE_PCI << 8;
	423	}
	424	DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
	425
82ba129b BH	426	static int pnv_pci_probe_mode(struct pci_bus *bus)
	427	{
	428	struct pci_controller *hose = pci_bus_to_host(bus);
	429	const __be64 *tstamp;
	430	u64 now, target;
	431
	432
	433	/* We hijack this as a way to ensure we have waited long
	434	* enough since the reset was lifted on the PCI bus
	435	*/
	436	if (bus != hose->bus)
	437	return PCI_PROBE_NORMAL;
	438	tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
	439	if (!tstamp \|\| !*tstamp)
	440	return PCI_PROBE_NORMAL;
	441
	442	now = mftb() / tb_ticks_per_usec;
	443	target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
	444	+ PCI_RESET_DELAY_US;
	445
	446	pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
	447	hose->global_number, target, now);
	448
	449	if (now < target)
	450	msleep((target - now + 999) / 1000);
	451
	452	return PCI_PROBE_NORMAL;
	453	}
	454
61305a96 BH	455	void __init pnv_pci_init(void)
	456	{
	457	struct device_node *np;
	458
	459	pci_set_flags(PCI_CAN_SKIP_ISA_ALIGN);
	460
	461	/* We do not want to just probe */
	462	pci_probe_only = 0;
	463
	464	/* OPAL absent, try POPAL first then RTAS detection of PHBs */
	465	if (!firmware_has_feature(FW_FEATURE_OPAL)) {
	466	#ifdef CONFIG_PPC_POWERNV_RTAS
	467	init_pci_config_tokens();
	468	find_and_init_phbs();
	469	#endif /* CONFIG_PPC_POWERNV_RTAS */
	470	} else {
	471	/* OPAL is here, do our normal stuff */
	472
	473	/* Look for p5ioc2 IO-Hubs */
	474	for_each_compatible_node(np, NULL, "ibm,p5ioc2")
	475	pnv_pci_init_p5ioc2_hub(np);
	476	}
	477
	478	/* Setup the linkage between OF nodes and PHBs */
	479	pci_devs_phb_init();
	480
	481	/* Configure IOMMU DMA hooks */
	482	ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
	483	ppc_md.tce_build = pnv_tce_build;
	484	ppc_md.tce_free = pnv_tce_free;
82ba129b	485	ppc_md.pci_probe_mode = pnv_pci_probe_mode;
61305a96 BH	486	set_pci_dma_ops(&dma_iommu_ops);
61305a96 BH	487
c1a2562a BH	488	/* Configure MSIs */
	489	#ifdef CONFIG_PCI_MSI
	490	ppc_md.msi_check_device = pnv_msi_check_device;
	491	ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
	492	ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
	493	#endif
61305a96	494	}