[deliverable/linux.git] / drivers / pci / iov.c

/*
 * drivers/pci/iov.c
 *
 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
 *
 * PCI Express I/O Virtualization (IOV) support.
 *   Single Root IOV 1.0
 */

#include <linux/pci.h>
#include <linux/mutex.h>
#include <linux/string.h>
#include <linux/delay.h>
#include "pci.h"

#define VIRTFN_ID_LEN	16

static inline u8 virtfn_bus(struct pci_dev *dev, int id)
{
	return dev->bus->number + ((dev->devfn + dev->sriov->offset +
				    dev->sriov->stride * id) >> 8);
}

static inline u8 virtfn_devfn(struct pci_dev *dev, int id)
{
	return (dev->devfn + dev->sriov->offset +
		dev->sriov->stride * id) & 0xff;
}

static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr)
{
	int rc;
	struct pci_bus *child;

	if (bus->number == busnr)
		return bus;

	child = pci_find_bus(pci_domain_nr(bus), busnr);
	if (child)
		return child;

	child = pci_add_new_bus(bus, NULL, busnr);
	if (!child)
		return NULL;

	child->subordinate = busnr;
	child->dev.parent = bus->bridge;
	rc = pci_bus_add_child(child);
	if (rc) {
		pci_remove_bus(child);
		return NULL;
	}

	return child;
}

static void virtfn_remove_bus(struct pci_bus *bus, int busnr)
{
	struct pci_bus *child;

	if (bus->number == busnr)
		return;

	child = pci_find_bus(pci_domain_nr(bus), busnr);
	BUG_ON(!child);

	if (list_empty(&child->devices))
		pci_remove_bus(child);
}

static int virtfn_add(struct pci_dev *dev, int id, int reset)
{
	int i;
	int rc;
	u64 size;
	char buf[VIRTFN_ID_LEN];
	struct pci_dev *virtfn;
	struct resource *res;
	struct pci_sriov *iov = dev->sriov;

	virtfn = alloc_pci_dev();
	if (!virtfn)
		return -ENOMEM;

	mutex_lock(&iov->dev->sriov->lock);
	virtfn->bus = virtfn_add_bus(dev->bus, virtfn_bus(dev, id));
	if (!virtfn->bus) {
		kfree(virtfn);
		mutex_unlock(&iov->dev->sriov->lock);
		return -ENOMEM;
	}
	virtfn->devfn = virtfn_devfn(dev, id);
	virtfn->vendor = dev->vendor;
	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
	pci_setup_device(virtfn);
	virtfn->dev.parent = dev->dev.parent;

	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
		res = dev->resource + PCI_IOV_RESOURCES + i;
		if (!res->parent)
			continue;
		virtfn->resource[i].name = pci_name(virtfn);
		virtfn->resource[i].flags = res->flags;
		size = resource_size(res);
		do_div(size, iov->total);
		virtfn->resource[i].start = res->start + size * id;
		virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
		rc = request_resource(res, &virtfn->resource[i]);
		BUG_ON(rc);
	}

	if (reset)
		pci_execute_reset_function(virtfn);

	pci_device_add(virtfn, virtfn->bus);
	mutex_unlock(&iov->dev->sriov->lock);

	virtfn->physfn = pci_dev_get(dev);
	virtfn->is_virtfn = 1;

	rc = pci_bus_add_device(virtfn);
	if (rc)
		goto failed1;
	sprintf(buf, "virtfn%u", id);
	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
	if (rc)
		goto failed1;
	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
	if (rc)
		goto failed2;

	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);

	return 0;

failed2:
	sysfs_remove_link(&dev->dev.kobj, buf);
failed1:
	pci_dev_put(dev);
	mutex_lock(&iov->dev->sriov->lock);
	pci_remove_bus_device(virtfn);
	virtfn_remove_bus(dev->bus, virtfn_bus(dev, id));
	mutex_unlock(&iov->dev->sriov->lock);

	return rc;
}

static void virtfn_remove(struct pci_dev *dev, int id, int reset)
{
	char buf[VIRTFN_ID_LEN];
	struct pci_bus *bus;
	struct pci_dev *virtfn;
	struct pci_sriov *iov = dev->sriov;

	bus = pci_find_bus(pci_domain_nr(dev->bus), virtfn_bus(dev, id));
	if (!bus)
		return;

	virtfn = pci_get_slot(bus, virtfn_devfn(dev, id));
	if (!virtfn)
		return;

	pci_dev_put(virtfn);

	if (reset) {
		device_release_driver(&virtfn->dev);
		pci_execute_reset_function(virtfn);
	}

	sprintf(buf, "virtfn%u", id);
	sysfs_remove_link(&dev->dev.kobj, buf);
	sysfs_remove_link(&virtfn->dev.kobj, "physfn");

	mutex_lock(&iov->dev->sriov->lock);
	pci_remove_bus_device(virtfn);
	virtfn_remove_bus(dev->bus, virtfn_bus(dev, id));
	mutex_unlock(&iov->dev->sriov->lock);

	pci_dev_put(dev);
}

static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
{
	int rc;
	int i, j;
	int nres;
	u16 offset, stride, initial;
	struct resource *res;
	struct pci_dev *pdev;
	struct pci_sriov *iov = dev->sriov;

	if (!nr_virtfn)
		return 0;

	if (iov->nr_virtfn)
		return -EINVAL;

	pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);
	if (initial > iov->total ||
	    (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total)))
		return -EIO;

	if (nr_virtfn < 0 || nr_virtfn > iov->total ||
	    (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial)))
		return -EINVAL;

	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);
	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &offset);
	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &stride);
	if (!offset || (nr_virtfn > 1 && !stride))
		return -EIO;

	nres = 0;
	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
		res = dev->resource + PCI_IOV_RESOURCES + i;
		if (res->parent)
			nres++;
	}
	if (nres != iov->nres) {
		dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n");
		return -ENOMEM;
	}

	iov->offset = offset;
	iov->stride = stride;

	if (virtfn_bus(dev, nr_virtfn - 1) > dev->bus->subordinate) {
		dev_err(&dev->dev, "SR-IOV: bus number out of range\n");
		return -ENOMEM;
	}

	if (iov->link != dev->devfn) {
		pdev = pci_get_slot(dev->bus, iov->link);
		if (!pdev)
			return -ENODEV;

		pci_dev_put(pdev);

		if (!pdev->is_physfn)
			return -ENODEV;

		rc = sysfs_create_link(&dev->dev.kobj,
					&pdev->dev.kobj, "dep_link");
		if (rc)
			return rc;
	}

	iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
	pci_block_user_cfg_access(dev);
	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
	msleep(100);
	pci_unblock_user_cfg_access(dev);

	iov->initial = initial;
	if (nr_virtfn < initial)
		initial = nr_virtfn;

	for (i = 0; i < initial; i++) {
		rc = virtfn_add(dev, i, 0);
		if (rc)
			goto failed;
	}

	kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
	iov->nr_virtfn = nr_virtfn;

	return 0;

failed:
	for (j = 0; j < i; j++)
		virtfn_remove(dev, j, 0);

	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
	pci_block_user_cfg_access(dev);
	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
	ssleep(1);
	pci_unblock_user_cfg_access(dev);

	if (iov->link != dev->devfn)
		sysfs_remove_link(&dev->dev.kobj, "dep_link");

	return rc;
}

static void sriov_disable(struct pci_dev *dev)
{
	int i;
	struct pci_sriov *iov = dev->sriov;

	if (!iov->nr_virtfn)
		return;

	for (i = 0; i < iov->nr_virtfn; i++)
		virtfn_remove(dev, i, 0);

	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
	pci_block_user_cfg_access(dev);
	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
	ssleep(1);
	pci_unblock_user_cfg_access(dev);

	if (iov->link != dev->devfn)
		sysfs_remove_link(&dev->dev.kobj, "dep_link");

	iov->nr_virtfn = 0;
}

static int sriov_init(struct pci_dev *dev, int pos)
{
	int i;
	int rc;
	int nres;
	u32 pgsz;
	u16 ctrl, total, offset, stride;
	struct pci_sriov *iov;
	struct resource *res;
	struct pci_dev *pdev;

	if (dev->pcie_type != PCI_EXP_TYPE_RC_END &&
	    dev->pcie_type != PCI_EXP_TYPE_ENDPOINT)
		return -ENODEV;

	pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);
	if (ctrl & PCI_SRIOV_CTRL_VFE) {
		pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);
		ssleep(1);
	}

	pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);
	if (!total)
		return 0;

	ctrl = 0;
	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
		if (pdev->is_physfn)
			goto found;

	pdev = NULL;
	if (pci_ari_enabled(dev->bus))
		ctrl |= PCI_SRIOV_CTRL_ARI;

found:
	pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);
	pci_write_config_word(dev, pos + PCI_SRIOV_NUM_VF, total);
	pci_read_config_word(dev, pos + PCI_SRIOV_VF_OFFSET, &offset);
	pci_read_config_word(dev, pos + PCI_SRIOV_VF_STRIDE, &stride);
	if (!offset || (total > 1 && !stride))
		return -EIO;

	pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);
	i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0;
	pgsz &= ~((1 << i) - 1);
	if (!pgsz)
		return -EIO;

	pgsz &= ~(pgsz - 1);
	pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);

	nres = 0;
	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
		res = dev->resource + PCI_IOV_RESOURCES + i;
		i += __pci_read_base(dev, pci_bar_unknown, res,
				     pos + PCI_SRIOV_BAR + i * 4);
		if (!res->flags)
			continue;
		if (resource_size(res) & (PAGE_SIZE - 1)) {
			rc = -EIO;
			goto failed;
		}
		res->end = res->start + resource_size(res) * total - 1;
		nres++;
	}

	iov = kzalloc(sizeof(*iov), GFP_KERNEL);
	if (!iov) {
		rc = -ENOMEM;
		goto failed;
	}

	iov->pos = pos;
	iov->nres = nres;
	iov->ctrl = ctrl;
	iov->total = total;
	iov->offset = offset;
	iov->stride = stride;
	iov->pgsz = pgsz;
	iov->self = dev;
	pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
	pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);

	if (pdev)
		iov->dev = pci_dev_get(pdev);
	else {
		iov->dev = dev;
		mutex_init(&iov->lock);
	}

	dev->sriov = iov;
	dev->is_physfn = 1;

	return 0;

failed:
	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
		res = dev->resource + PCI_IOV_RESOURCES + i;
		res->flags = 0;
	}

	return rc;
}

static void sriov_release(struct pci_dev *dev)
{
	BUG_ON(dev->sriov->nr_virtfn);

	if (dev == dev->sriov->dev)
		mutex_destroy(&dev->sriov->lock);
	else
		pci_dev_put(dev->sriov->dev);

	kfree(dev->sriov);
	dev->sriov = NULL;
}

static void sriov_restore_state(struct pci_dev *dev)
{
	int i;
	u16 ctrl;
	struct pci_sriov *iov = dev->sriov;

	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);
	if (ctrl & PCI_SRIOV_CTRL_VFE)
		return;

	for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++)
		pci_update_resource(dev, i);

	pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, iov->nr_virtfn);
	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
	if (iov->ctrl & PCI_SRIOV_CTRL_VFE)
		msleep(100);
}

/**
 * pci_iov_init - initialize the IOV capability
 * @dev: the PCI device
 *
 * Returns 0 on success, or negative on failure.
 */
int pci_iov_init(struct pci_dev *dev)
{
	int pos;

	if (!dev->is_pcie)
		return -ENODEV;

	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
	if (pos)
		return sriov_init(dev, pos);

	return -ENODEV;
}

/**
 * pci_iov_release - release resources used by the IOV capability
 * @dev: the PCI device
 */
void pci_iov_release(struct pci_dev *dev)
{
	if (dev->is_physfn)
		sriov_release(dev);
}

/**
 * pci_iov_resource_bar - get position of the SR-IOV BAR
 * @dev: the PCI device
 * @resno: the resource number
 * @type: the BAR type to be filled in
 *
 * Returns position of the BAR encapsulated in the SR-IOV capability.
 */
int pci_iov_resource_bar(struct pci_dev *dev, int resno,
			 enum pci_bar_type *type)
{
	if (resno < PCI_IOV_RESOURCES || resno > PCI_IOV_RESOURCE_END)
		return 0;

	BUG_ON(!dev->is_physfn);

	*type = pci_bar_unknown;

	return dev->sriov->pos + PCI_SRIOV_BAR +
		4 * (resno - PCI_IOV_RESOURCES);
}

/**
 * pci_restore_iov_state - restore the state of the IOV capability
 * @dev: the PCI device
 */
void pci_restore_iov_state(struct pci_dev *dev)
{
	if (dev->is_physfn)
		sriov_restore_state(dev);
}

/**
 * pci_iov_bus_range - find bus range used by Virtual Function
 * @bus: the PCI bus
 *
 * Returns max number of buses (exclude current one) used by Virtual
 * Functions.
 */
int pci_iov_bus_range(struct pci_bus *bus)
{
	int max = 0;
	u8 busnr;
	struct pci_dev *dev;

	list_for_each_entry(dev, &bus->devices, bus_list) {
		if (!dev->is_physfn)
			continue;
		busnr = virtfn_bus(dev, dev->sriov->total - 1);
		if (busnr > max)
			max = busnr;
	}

	return max ? max - bus->number : 0;
}

/**
 * pci_enable_sriov - enable the SR-IOV capability
 * @dev: the PCI device
 *
 * Returns 0 on success, or negative on failure.
 */
int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
{
	might_sleep();

	if (!dev->is_physfn)
		return -ENODEV;

	return sriov_enable(dev, nr_virtfn);
}
EXPORT_SYMBOL_GPL(pci_enable_sriov);

/**
 * pci_disable_sriov - disable the SR-IOV capability
 * @dev: the PCI device
 */
void pci_disable_sriov(struct pci_dev *dev)
{
	might_sleep();

	if (!dev->is_physfn)
		return;

	sriov_disable(dev);
}
EXPORT_SYMBOL_GPL(pci_disable_sriov);
Commit	Line	Data
d1b054da YZ	1	/*
	2	* drivers/pci/iov.c
	3	*
	4	* Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
	5	*
	6	* PCI Express I/O Virtualization (IOV) support.
	7	* Single Root IOV 1.0
	8	*/
	9
	10	#include <linux/pci.h>
	11	#include <linux/mutex.h>
	12	#include <linux/string.h>
	13	#include <linux/delay.h>
	14	#include "pci.h"
	15
dd7cc44d	16	#define VIRTFN_ID_LEN 16
d1b054da	17
a28724b0 YZ	18	static inline u8 virtfn_bus(struct pci_dev *dev, int id)
	19	{
	20	return dev->bus->number + ((dev->devfn + dev->sriov->offset +
	21	dev->sriov->stride * id) >> 8);
	22	}
	23
	24	static inline u8 virtfn_devfn(struct pci_dev *dev, int id)
	25	{
	26	return (dev->devfn + dev->sriov->offset +
	27	dev->sriov->stride * id) & 0xff;
	28	}
	29
dd7cc44d YZ	30	static struct pci_bus virtfn_add_bus(struct pci_bus bus, int busnr)
	31	{
	32	int rc;
	33	struct pci_bus *child;
	34
	35	if (bus->number == busnr)
	36	return bus;
	37
	38	child = pci_find_bus(pci_domain_nr(bus), busnr);
	39	if (child)
	40	return child;
	41
	42	child = pci_add_new_bus(bus, NULL, busnr);
	43	if (!child)
	44	return NULL;
	45
	46	child->subordinate = busnr;
	47	child->dev.parent = bus->bridge;
	48	rc = pci_bus_add_child(child);
	49	if (rc) {
	50	pci_remove_bus(child);
	51	return NULL;
	52	}
	53
	54	return child;
	55	}
	56
	57	static void virtfn_remove_bus(struct pci_bus *bus, int busnr)
	58	{
	59	struct pci_bus *child;
	60
	61	if (bus->number == busnr)
	62	return;
	63
	64	child = pci_find_bus(pci_domain_nr(bus), busnr);
	65	BUG_ON(!child);
	66
	67	if (list_empty(&child->devices))
	68	pci_remove_bus(child);
	69	}
	70
	71	static int virtfn_add(struct pci_dev *dev, int id, int reset)
	72	{
	73	int i;
	74	int rc;
	75	u64 size;
	76	char buf[VIRTFN_ID_LEN];
	77	struct pci_dev *virtfn;
	78	struct resource *res;
	79	struct pci_sriov *iov = dev->sriov;
	80
	81	virtfn = alloc_pci_dev();
	82	if (!virtfn)
	83	return -ENOMEM;
	84
	85	mutex_lock(&iov->dev->sriov->lock);
	86	virtfn->bus = virtfn_add_bus(dev->bus, virtfn_bus(dev, id));
	87	if (!virtfn->bus) {
	88	kfree(virtfn);
	89	mutex_unlock(&iov->dev->sriov->lock);
	90	return -ENOMEM;
	91	}
	92	virtfn->devfn = virtfn_devfn(dev, id);
	93	virtfn->vendor = dev->vendor;
94	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
95	pci_setup_device(virtfn);
96	virtfn->dev.parent = dev->dev.parent;
97
98	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
99	res = dev->resource + PCI_IOV_RESOURCES + i;
100	if (!res->parent)
101	continue;
102	virtfn->resource[i].name = pci_name(virtfn);
103	virtfn->resource[i].flags = res->flags;
104	size = resource_size(res);
105	do_div(size, iov->total);
106	virtfn->resource[i].start = res->start + size * id;
107	virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
108	rc = request_resource(res, &virtfn->resource[i]);
109	BUG_ON(rc);
110	}
111
112	if (reset)
113	pci_execute_reset_function(virtfn);
114
115	pci_device_add(virtfn, virtfn->bus);
116	mutex_unlock(&iov->dev->sriov->lock);
117
118	virtfn->physfn = pci_dev_get(dev);
119	virtfn->is_virtfn = 1;
120
121	rc = pci_bus_add_device(virtfn);
122	if (rc)
123	goto failed1;
124	sprintf(buf, "virtfn%u", id);
125	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
126	if (rc)
127	goto failed1;
128	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
129	if (rc)
130	goto failed2;
131
132	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
133
134	return 0;
135
136	failed2:
137	sysfs_remove_link(&dev->dev.kobj, buf);
138	failed1:
139	pci_dev_put(dev);
140	mutex_lock(&iov->dev->sriov->lock);
141	pci_remove_bus_device(virtfn);
142	virtfn_remove_bus(dev->bus, virtfn_bus(dev, id));
143	mutex_unlock(&iov->dev->sriov->lock);
144
145	return rc;
146	}
147
148	static void virtfn_remove(struct pci_dev *dev, int id, int reset)
149	{
150	char buf[VIRTFN_ID_LEN];
151	struct pci_bus *bus;
152	struct pci_dev *virtfn;
153	struct pci_sriov *iov = dev->sriov;
154
155	bus = pci_find_bus(pci_domain_nr(dev->bus), virtfn_bus(dev, id));
156	if (!bus)
157	return;
158
159	virtfn = pci_get_slot(bus, virtfn_devfn(dev, id));
160	if (!virtfn)
161	return;
162
163	pci_dev_put(virtfn);
164
165	if (reset) {
166	device_release_driver(&virtfn->dev);
167	pci_execute_reset_function(virtfn);
168	}
169
170	sprintf(buf, "virtfn%u", id);
171	sysfs_remove_link(&dev->dev.kobj, buf);
172	sysfs_remove_link(&virtfn->dev.kobj, "physfn");
173
174	mutex_lock(&iov->dev->sriov->lock);
175	pci_remove_bus_device(virtfn);
176	virtfn_remove_bus(dev->bus, virtfn_bus(dev, id));
177	mutex_unlock(&iov->dev->sriov->lock);
178
179	pci_dev_put(dev);
180	}
181
182	static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
183	{
184	int rc;
185	int i, j;
186	int nres;
187	u16 offset, stride, initial;
188	struct resource *res;
189	struct pci_dev *pdev;
190	struct pci_sriov *iov = dev->sriov;
191
192	if (!nr_virtfn)
193	return 0;
194
195	if (iov->nr_virtfn)
196	return -EINVAL;
197
198	pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);
199	if (initial > iov->total \|\|
200	(!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total)))
201	return -EIO;
202
203	if (nr_virtfn < 0 \|\| nr_virtfn > iov->total \|\|
204	(!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial)))
205	return -EINVAL;
206
207	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);
208	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &offset);
209	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &stride);
210	if (!offset \|\| (nr_virtfn > 1 && !stride))
211	return -EIO;
212
213	nres = 0;
214	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
215	res = dev->resource + PCI_IOV_RESOURCES + i;
216	if (res->parent)
217	nres++;
218	}
219	if (nres != iov->nres) {
220	dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n");
221	return -ENOMEM;
222	}
223
224	iov->offset = offset;
225	iov->stride = stride;
226
227	if (virtfn_bus(dev, nr_virtfn - 1) > dev->bus->subordinate) {
228	dev_err(&dev->dev, "SR-IOV: bus number out of range\n");
229	return -ENOMEM;
230	}
231
232	if (iov->link != dev->devfn) {
233	pdev = pci_get_slot(dev->bus, iov->link);
234	if (!pdev)
235	return -ENODEV;
236
237	pci_dev_put(pdev);
238
239	if (!pdev->is_physfn)
240	return -ENODEV;
241
242	rc = sysfs_create_link(&dev->dev.kobj,
243	&pdev->dev.kobj, "dep_link");
244	if (rc)
245	return rc;
246	}
247
248	iov->ctrl \|= PCI_SRIOV_CTRL_VFE \| PCI_SRIOV_CTRL_MSE;
249	pci_block_user_cfg_access(dev);
250	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
251	msleep(100);
252	pci_unblock_user_cfg_access(dev);
253
254	iov->initial = initial;
255	if (nr_virtfn < initial)
256	initial = nr_virtfn;
257
258	for (i = 0; i < initial; i++) {
259	rc = virtfn_add(dev, i, 0);
260	if (rc)
261	goto failed;
262	}
263
264	kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
265	iov->nr_virtfn = nr_virtfn;
266
267	return 0;
268
269	failed:
270	for (j = 0; j < i; j++)
271	virtfn_remove(dev, j, 0);
272
273	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE \| PCI_SRIOV_CTRL_MSE);
274	pci_block_user_cfg_access(dev);
275	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
276	ssleep(1);
277	pci_unblock_user_cfg_access(dev);
278
279	if (iov->link != dev->devfn)
280	sysfs_remove_link(&dev->dev.kobj, "dep_link");
281
282	return rc;
283	}
284
285	static void sriov_disable(struct pci_dev *dev)
286	{
287	int i;
288	struct pci_sriov *iov = dev->sriov;
289
290	if (!iov->nr_virtfn)
291	return;
292
293	for (i = 0; i < iov->nr_virtfn; i++)
294	virtfn_remove(dev, i, 0);
295
296	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE \| PCI_SRIOV_CTRL_MSE);
297	pci_block_user_cfg_access(dev);
298	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
299	ssleep(1);
300	pci_unblock_user_cfg_access(dev);
301
302	if (iov->link != dev->devfn)
303	sysfs_remove_link(&dev->dev.kobj, "dep_link");
304
305	iov->nr_virtfn = 0;
306	}
307
d1b054da YZ	308	static int sriov_init(struct pci_dev *dev, int pos)
	309	{
	310	int i;
	311	int rc;
	312	int nres;
	313	u32 pgsz;
	314	u16 ctrl, total, offset, stride;
	315	struct pci_sriov *iov;
	316	struct resource *res;
	317	struct pci_dev *pdev;
	318
	319	if (dev->pcie_type != PCI_EXP_TYPE_RC_END &&
	320	dev->pcie_type != PCI_EXP_TYPE_ENDPOINT)
	321	return -ENODEV;
	322
	323	pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);
	324	if (ctrl & PCI_SRIOV_CTRL_VFE) {
	325	pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);
	326	ssleep(1);
	327	}
	328
	329	pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);
	330	if (!total)
	331	return 0;
	332
	333	ctrl = 0;
	334	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
	335	if (pdev->is_physfn)
	336	goto found;
	337
	338	pdev = NULL;
	339	if (pci_ari_enabled(dev->bus))
	340	ctrl \|= PCI_SRIOV_CTRL_ARI;
	341
	342	found:
	343	pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);
	344	pci_write_config_word(dev, pos + PCI_SRIOV_NUM_VF, total);
	345	pci_read_config_word(dev, pos + PCI_SRIOV_VF_OFFSET, &offset);
	346	pci_read_config_word(dev, pos + PCI_SRIOV_VF_STRIDE, &stride);
	347	if (!offset \|\| (total > 1 && !stride))
	348	return -EIO;
	349
	350	pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);
	351	i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0;
	352	pgsz &= ~((1 << i) - 1);
	353	if (!pgsz)
	354	return -EIO;
	355
	356	pgsz &= ~(pgsz - 1);
	357	pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);
	358
	359	nres = 0;
	360	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
	361	res = dev->resource + PCI_IOV_RESOURCES + i;
	362	i += __pci_read_base(dev, pci_bar_unknown, res,
	363	pos + PCI_SRIOV_BAR + i * 4);
	364	if (!res->flags)
	365	continue;
	366	if (resource_size(res) & (PAGE_SIZE - 1)) {
	367	rc = -EIO;
	368	goto failed;
	369	}
	370	res->end = res->start + resource_size(res) * total - 1;
	371	nres++;
372	}
373
374	iov = kzalloc(sizeof(*iov), GFP_KERNEL);
375	if (!iov) {
376	rc = -ENOMEM;
377	goto failed;
378	}
379
380	iov->pos = pos;
381	iov->nres = nres;
382	iov->ctrl = ctrl;
383	iov->total = total;
384	iov->offset = offset;
385	iov->stride = stride;
386	iov->pgsz = pgsz;
387	iov->self = dev;
388	pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
389	pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
390
391	if (pdev)
392	iov->dev = pci_dev_get(pdev);
393	else {
394	iov->dev = dev;
395	mutex_init(&iov->lock);
396	}
397
398	dev->sriov = iov;
399	dev->is_physfn = 1;
400
401	return 0;
402
403	failed:
404	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
405	res = dev->resource + PCI_IOV_RESOURCES + i;
406	res->flags = 0;
407	}
408
409	return rc;
410	}
411
412	static void sriov_release(struct pci_dev *dev)
413	{
dd7cc44d YZ	414	BUG_ON(dev->sriov->nr_virtfn);
dd7cc44d YZ	415
d1b054da YZ	416	if (dev == dev->sriov->dev)
	417	mutex_destroy(&dev->sriov->lock);
	418	else
	419	pci_dev_put(dev->sriov->dev);
	420
	421	kfree(dev->sriov);
	422	dev->sriov = NULL;
	423	}
	424
8c5cdb6a YZ	425	static void sriov_restore_state(struct pci_dev *dev)
	426	{
	427	int i;
	428	u16 ctrl;
	429	struct pci_sriov *iov = dev->sriov;
	430
	431	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);
	432	if (ctrl & PCI_SRIOV_CTRL_VFE)
	433	return;
	434
	435	for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++)
	436	pci_update_resource(dev, i);
	437
	438	pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
dd7cc44d	439	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, iov->nr_virtfn);
8c5cdb6a YZ	440	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
	441	if (iov->ctrl & PCI_SRIOV_CTRL_VFE)
	442	msleep(100);
	443	}
	444
d1b054da YZ	445	/**
	446	* pci_iov_init - initialize the IOV capability
	447	* @dev: the PCI device
	448	*
	449	* Returns 0 on success, or negative on failure.
	450	*/
	451	int pci_iov_init(struct pci_dev *dev)
	452	{
	453	int pos;
	454
	455	if (!dev->is_pcie)
	456	return -ENODEV;
	457
	458	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
	459	if (pos)
	460	return sriov_init(dev, pos);
	461
	462	return -ENODEV;
	463	}
	464
	465	/**
	466	* pci_iov_release - release resources used by the IOV capability
	467	* @dev: the PCI device
	468	*/
	469	void pci_iov_release(struct pci_dev *dev)
	470	{
	471	if (dev->is_physfn)
	472	sriov_release(dev);
	473	}
	474
	475	/**
	476	* pci_iov_resource_bar - get position of the SR-IOV BAR
	477	* @dev: the PCI device
	478	* @resno: the resource number
	479	* @type: the BAR type to be filled in
	480	*
	481	* Returns position of the BAR encapsulated in the SR-IOV capability.
	482	*/
	483	int pci_iov_resource_bar(struct pci_dev *dev, int resno,
	484	enum pci_bar_type *type)
	485	{
	486	if (resno < PCI_IOV_RESOURCES \|\| resno > PCI_IOV_RESOURCE_END)
	487	return 0;
	488
	489	BUG_ON(!dev->is_physfn);
	490
	491	*type = pci_bar_unknown;
	492
	493	return dev->sriov->pos + PCI_SRIOV_BAR +
	494	4 * (resno - PCI_IOV_RESOURCES);
	495	}
8c5cdb6a YZ	496
	497	/**
	498	* pci_restore_iov_state - restore the state of the IOV capability
	499	* @dev: the PCI device
	500	*/
	501	void pci_restore_iov_state(struct pci_dev *dev)
	502	{
	503	if (dev->is_physfn)
	504	sriov_restore_state(dev);
	505	}
a28724b0 YZ	506
	507	/**
	508	* pci_iov_bus_range - find bus range used by Virtual Function
	509	* @bus: the PCI bus
	510	*
	511	* Returns max number of buses (exclude current one) used by Virtual
	512	* Functions.
	513	*/
	514	int pci_iov_bus_range(struct pci_bus *bus)
	515	{
	516	int max = 0;
	517	u8 busnr;
	518	struct pci_dev *dev;
	519
	520	list_for_each_entry(dev, &bus->devices, bus_list) {
	521	if (!dev->is_physfn)
	522	continue;
	523	busnr = virtfn_bus(dev, dev->sriov->total - 1);
	524	if (busnr > max)
	525	max = busnr;
	526	}
	527
	528	return max ? max - bus->number : 0;
	529	}
dd7cc44d YZ	530
	531	/**
	532	* pci_enable_sriov - enable the SR-IOV capability
	533	* @dev: the PCI device
	534	*
	535	* Returns 0 on success, or negative on failure.
	536	*/
	537	int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
	538	{
	539	might_sleep();
	540
	541	if (!dev->is_physfn)
	542	return -ENODEV;
	543
	544	return sriov_enable(dev, nr_virtfn);
	545	}
	546	EXPORT_SYMBOL_GPL(pci_enable_sriov);
	547
	548	/**
	549	* pci_disable_sriov - disable the SR-IOV capability
	550	* @dev: the PCI device
	551	*/
	552	void pci_disable_sriov(struct pci_dev *dev)
	553	{
	554	might_sleep();
	555
	556	if (!dev->is_physfn)
	557	return;
	558
	559	sriov_disable(dev);
	560	}
	561	EXPORT_SYMBOL_GPL(pci_disable_sriov);