[deliverable/linux.git] / drivers / iommu / iommu.c

/*
 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
 * Author: Joerg Roedel <joerg.roedel@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#define pr_fmt(fmt)    "%s: " fmt, __func__

#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/iommu.h>

static ssize_t show_iommu_group(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	unsigned int groupid;

	if (iommu_device_group(dev, &groupid))
		return 0;

	return sprintf(buf, "%u", groupid);
}
static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL);

static int add_iommu_group(struct device *dev, void *data)
{
	unsigned int groupid;

	if (iommu_device_group(dev, &groupid) == 0)
		return device_create_file(dev, &dev_attr_iommu_group);

	return 0;
}

static int remove_iommu_group(struct device *dev)
{
	unsigned int groupid;

	if (iommu_device_group(dev, &groupid) == 0)
		device_remove_file(dev, &dev_attr_iommu_group);

	return 0;
}

static int iommu_device_notifier(struct notifier_block *nb,
				 unsigned long action, void *data)
{
	struct device *dev = data;

	if (action == BUS_NOTIFY_ADD_DEVICE)
		return add_iommu_group(dev, NULL);
	else if (action == BUS_NOTIFY_DEL_DEVICE)
		return remove_iommu_group(dev);

	return 0;
}

static struct notifier_block iommu_device_nb = {
	.notifier_call = iommu_device_notifier,
};

static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops)
{
	bus_register_notifier(bus, &iommu_device_nb);
	bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
}

/**
 * bus_set_iommu - set iommu-callbacks for the bus
 * @bus: bus.
 * @ops: the callbacks provided by the iommu-driver
 *
 * This function is called by an iommu driver to set the iommu methods
 * used for a particular bus. Drivers for devices on that bus can use
 * the iommu-api after these ops are registered.
 * This special function is needed because IOMMUs are usually devices on
 * the bus itself, so the iommu drivers are not initialized when the bus
 * is set up. With this function the iommu-driver can set the iommu-ops
 * afterwards.
 */
int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops)
{
	if (bus->iommu_ops != NULL)
		return -EBUSY;

	bus->iommu_ops = ops;

	/* Do IOMMU specific setup for this bus-type */
	iommu_bus_init(bus, ops);

	return 0;
}
EXPORT_SYMBOL_GPL(bus_set_iommu);

bool iommu_present(struct bus_type *bus)
{
	return bus->iommu_ops != NULL;
}
EXPORT_SYMBOL_GPL(iommu_present);

/**
 * iommu_set_fault_handler() - set a fault handler for an iommu domain
 * @domain: iommu domain
 * @handler: fault handler
 *
 * This function should be used by IOMMU users which want to be notified
 * whenever an IOMMU fault happens.
 *
 * The fault handler itself should return 0 on success, and an appropriate
 * error code otherwise.
 */
void iommu_set_fault_handler(struct iommu_domain *domain,
					iommu_fault_handler_t handler)
{
	BUG_ON(!domain);

	domain->handler = handler;
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);

struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
{
	struct iommu_domain *domain;
	int ret;

	if (bus == NULL || bus->iommu_ops == NULL)
		return NULL;

	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
	if (!domain)
		return NULL;

	domain->ops = bus->iommu_ops;

	ret = domain->ops->domain_init(domain);
	if (ret)
		goto out_free;

	return domain;

out_free:
	kfree(domain);

	return NULL;
}
EXPORT_SYMBOL_GPL(iommu_domain_alloc);

void iommu_domain_free(struct iommu_domain *domain)
{
	if (likely(domain->ops->domain_destroy != NULL))
		domain->ops->domain_destroy(domain);

	kfree(domain);
}
EXPORT_SYMBOL_GPL(iommu_domain_free);

int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
	if (unlikely(domain->ops->attach_dev == NULL))
		return -ENODEV;

	return domain->ops->attach_dev(domain, dev);
}
EXPORT_SYMBOL_GPL(iommu_attach_device);

void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
{
	if (unlikely(domain->ops->detach_dev == NULL))
		return;

	domain->ops->detach_dev(domain, dev);
}
EXPORT_SYMBOL_GPL(iommu_detach_device);

phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain,
			       unsigned long iova)
{
	if (unlikely(domain->ops->iova_to_phys == NULL))
		return 0;

	return domain->ops->iova_to_phys(domain, iova);
}
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);

int iommu_domain_has_cap(struct iommu_domain *domain,
			 unsigned long cap)
{
	if (unlikely(domain->ops->domain_has_cap == NULL))
		return 0;

	return domain->ops->domain_has_cap(domain, cap);
}
EXPORT_SYMBOL_GPL(iommu_domain_has_cap);

int iommu_map(struct iommu_domain *domain, unsigned long iova,
	      phys_addr_t paddr, size_t size, int prot)
{
	unsigned long orig_iova = iova;
	unsigned int min_pagesz;
	size_t orig_size = size;
	int ret = 0;

	if (unlikely(domain->ops->map == NULL))
		return -ENODEV;

	/* find out the minimum page size supported */
	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);

	/*
	 * both the virtual address and the physical one, as well as
	 * the size of the mapping, must be aligned (at least) to the
	 * size of the smallest page supported by the hardware
	 */
	if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
		pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz "
			"0x%x\n", iova, (unsigned long)paddr,
			(unsigned long)size, min_pagesz);
		return -EINVAL;
	}

	pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova,
				(unsigned long)paddr, (unsigned long)size);

	while (size) {
		unsigned long pgsize, addr_merge = iova | paddr;
		unsigned int pgsize_idx;

		/* Max page size that still fits into 'size' */
		pgsize_idx = __fls(size);

		/* need to consider alignment requirements ? */
		if (likely(addr_merge)) {
			/* Max page size allowed by both iova and paddr */
			unsigned int align_pgsize_idx = __ffs(addr_merge);

			pgsize_idx = min(pgsize_idx, align_pgsize_idx);
		}

		/* build a mask of acceptable page sizes */
		pgsize = (1UL << (pgsize_idx + 1)) - 1;

		/* throw away page sizes not supported by the hardware */
		pgsize &= domain->ops->pgsize_bitmap;

		/* make sure we're still sane */
		BUG_ON(!pgsize);

		/* pick the biggest page */
		pgsize_idx = __fls(pgsize);
		pgsize = 1UL << pgsize_idx;

		pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova,
					(unsigned long)paddr, pgsize);

		ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
		if (ret)
			break;

		iova += pgsize;
		paddr += pgsize;
		size -= pgsize;
	}

	/* unroll mapping in case something went wrong */
	if (ret)
		iommu_unmap(domain, orig_iova, orig_size - size);

	return ret;
}
EXPORT_SYMBOL_GPL(iommu_map);

size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
{
	size_t unmapped_page, unmapped = 0;
	unsigned int min_pagesz;

	if (unlikely(domain->ops->unmap == NULL))
		return -ENODEV;

	/* find out the minimum page size supported */
	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);

	/*
	 * The virtual address, as well as the size of the mapping, must be
	 * aligned (at least) to the size of the smallest page supported
	 * by the hardware
	 */
	if (!IS_ALIGNED(iova | size, min_pagesz)) {
		pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n",
					iova, (unsigned long)size, min_pagesz);
		return -EINVAL;
	}

	pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova,
							(unsigned long)size);

	/*
	 * Keep iterating until we either unmap 'size' bytes (or more)
	 * or we hit an area that isn't mapped.
	 */
	while (unmapped < size) {
		size_t left = size - unmapped;

		unmapped_page = domain->ops->unmap(domain, iova, left);
		if (!unmapped_page)
			break;

		pr_debug("unmapped: iova 0x%lx size %lx\n", iova,
					(unsigned long)unmapped_page);

		iova += unmapped_page;
		unmapped += unmapped_page;
	}

	return unmapped;
}
EXPORT_SYMBOL_GPL(iommu_unmap);

int iommu_device_group(struct device *dev, unsigned int *groupid)
{
	if (iommu_present(dev->bus) && dev->bus->iommu_ops->device_group)
		return dev->bus->iommu_ops->device_group(dev, groupid);

	return -ENODEV;
}
EXPORT_SYMBOL_GPL(iommu_device_group);
Commit	Line	Data
fc2100eb JR	1	/*
	2	* Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
	3	* Author: Joerg Roedel <joerg.roedel@amd.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License version 2 as published
	7	* by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program; if not, write to the Free Software
	16	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	17	*/
	18
7d3002cc OBC	19	#define pr_fmt(fmt) "%s: " fmt, __func__
7d3002cc OBC	20
905d66c1	21	#include <linux/device.h>
40998188	22	#include <linux/kernel.h>
fc2100eb JR	23	#include <linux/bug.h>
fc2100eb JR	24	#include <linux/types.h>
60db4027 AM	25	#include <linux/module.h>
60db4027 AM	26	#include <linux/slab.h>
fc2100eb JR	27	#include <linux/errno.h>
	28	#include <linux/iommu.h>
	29
1460432c AW	30	static ssize_t show_iommu_group(struct device *dev,
	31	struct device_attribute attr, char buf)
	32	{
	33	unsigned int groupid;
	34
	35	if (iommu_device_group(dev, &groupid))
	36	return 0;
	37
	38	return sprintf(buf, "%u", groupid);
	39	}
	40	static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL);
	41
	42	static int add_iommu_group(struct device dev, void data)
	43	{
	44	unsigned int groupid;
	45
	46	if (iommu_device_group(dev, &groupid) == 0)
	47	return device_create_file(dev, &dev_attr_iommu_group);
	48
	49	return 0;
	50	}
	51
	52	static int remove_iommu_group(struct device *dev)
	53	{
	54	unsigned int groupid;
	55
	56	if (iommu_device_group(dev, &groupid) == 0)
	57	device_remove_file(dev, &dev_attr_iommu_group);
	58
	59	return 0;
	60	}
	61
	62	static int iommu_device_notifier(struct notifier_block *nb,
	63	unsigned long action, void *data)
	64	{
	65	struct device *dev = data;
	66
	67	if (action == BUS_NOTIFY_ADD_DEVICE)
	68	return add_iommu_group(dev, NULL);
	69	else if (action == BUS_NOTIFY_DEL_DEVICE)
	70	return remove_iommu_group(dev);
	71
	72	return 0;
	73	}
	74
	75	static struct notifier_block iommu_device_nb = {
	76	.notifier_call = iommu_device_notifier,
	77	};
	78
ff21776d JR	79	static void iommu_bus_init(struct bus_type bus, struct iommu_ops ops)
ff21776d JR	80	{
1460432c AW	81	bus_register_notifier(bus, &iommu_device_nb);
1460432c AW	82	bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
ff21776d	83	}
fc2100eb	84
ff21776d JR	85	/**
	86	* bus_set_iommu - set iommu-callbacks for the bus
	87	* @bus: bus.
	88	* @ops: the callbacks provided by the iommu-driver
	89	*
	90	* This function is called by an iommu driver to set the iommu methods
	91	* used for a particular bus. Drivers for devices on that bus can use
	92	* the iommu-api after these ops are registered.
	93	* This special function is needed because IOMMUs are usually devices on
	94	* the bus itself, so the iommu drivers are not initialized when the bus
	95	* is set up. With this function the iommu-driver can set the iommu-ops
	96	* afterwards.
	97	*/
	98	int bus_set_iommu(struct bus_type bus, struct iommu_ops ops)
fc2100eb	99	{
ff21776d JR	100	if (bus->iommu_ops != NULL)
ff21776d JR	101	return -EBUSY;
fc2100eb	102
ff21776d JR	103	bus->iommu_ops = ops;
	104
	105	/* Do IOMMU specific setup for this bus-type */
	106	iommu_bus_init(bus, ops);
fc2100eb	107
ff21776d	108	return 0;
fc2100eb	109	}
ff21776d	110	EXPORT_SYMBOL_GPL(bus_set_iommu);
fc2100eb	111
a1b60c1c	112	bool iommu_present(struct bus_type *bus)
fc2100eb	113	{
94441c3b	114	return bus->iommu_ops != NULL;
fc2100eb	115	}
a1b60c1c	116	EXPORT_SYMBOL_GPL(iommu_present);
fc2100eb	117
4f3f8d9d OBC	118	/**
	119	* iommu_set_fault_handler() - set a fault handler for an iommu domain
	120	* @domain: iommu domain
	121	* @handler: fault handler
0ed6d2d2 OBC	122	*
	123	* This function should be used by IOMMU users which want to be notified
	124	* whenever an IOMMU fault happens.
	125	*
	126	* The fault handler itself should return 0 on success, and an appropriate
	127	* error code otherwise.
4f3f8d9d OBC	128	*/
	129	void iommu_set_fault_handler(struct iommu_domain *domain,
	130	iommu_fault_handler_t handler)
	131	{
	132	BUG_ON(!domain);
	133
	134	domain->handler = handler;
	135	}
30bd918c	136	EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
4f3f8d9d	137
905d66c1	138	struct iommu_domain iommu_domain_alloc(struct bus_type bus)
fc2100eb JR	139	{
	140	struct iommu_domain *domain;
	141	int ret;
	142
94441c3b	143	if (bus == NULL \|\| bus->iommu_ops == NULL)
905d66c1 JR	144	return NULL;
905d66c1 JR	145
8bd6960c	146	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
fc2100eb JR	147	if (!domain)
	148	return NULL;
	149
94441c3b	150	domain->ops = bus->iommu_ops;
905d66c1	151
94441c3b	152	ret = domain->ops->domain_init(domain);
fc2100eb JR	153	if (ret)
	154	goto out_free;
	155
	156	return domain;
	157
	158	out_free:
	159	kfree(domain);
	160
	161	return NULL;
	162	}
	163	EXPORT_SYMBOL_GPL(iommu_domain_alloc);
	164
	165	void iommu_domain_free(struct iommu_domain *domain)
	166	{
e5aa7f00 JR	167	if (likely(domain->ops->domain_destroy != NULL))
	168	domain->ops->domain_destroy(domain);
	169
fc2100eb JR	170	kfree(domain);
	171	}
	172	EXPORT_SYMBOL_GPL(iommu_domain_free);
	173
	174	int iommu_attach_device(struct iommu_domain domain, struct device dev)
	175	{
e5aa7f00 JR	176	if (unlikely(domain->ops->attach_dev == NULL))
	177	return -ENODEV;
	178
	179	return domain->ops->attach_dev(domain, dev);
fc2100eb JR	180	}
	181	EXPORT_SYMBOL_GPL(iommu_attach_device);
	182
	183	void iommu_detach_device(struct iommu_domain domain, struct device dev)
	184	{
e5aa7f00 JR	185	if (unlikely(domain->ops->detach_dev == NULL))
	186	return;
	187
	188	domain->ops->detach_dev(domain, dev);
fc2100eb JR	189	}
	190	EXPORT_SYMBOL_GPL(iommu_detach_device);
	191
fc2100eb JR	192	phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain,
	193	unsigned long iova)
	194	{
e5aa7f00 JR	195	if (unlikely(domain->ops->iova_to_phys == NULL))
	196	return 0;
	197
	198	return domain->ops->iova_to_phys(domain, iova);
fc2100eb JR	199	}
fc2100eb JR	200	EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
dbb9fd86 SY	201
	202	int iommu_domain_has_cap(struct iommu_domain *domain,
	203	unsigned long cap)
	204	{
e5aa7f00 JR	205	if (unlikely(domain->ops->domain_has_cap == NULL))
	206	return 0;
	207
	208	return domain->ops->domain_has_cap(domain, cap);
dbb9fd86 SY	209	}
dbb9fd86 SY	210	EXPORT_SYMBOL_GPL(iommu_domain_has_cap);
cefc53c7 JR	211
cefc53c7 JR	212	int iommu_map(struct iommu_domain *domain, unsigned long iova,
7d3002cc	213	phys_addr_t paddr, size_t size, int prot)
cefc53c7	214	{
7d3002cc OBC	215	unsigned long orig_iova = iova;
	216	unsigned int min_pagesz;
	217	size_t orig_size = size;
	218	int ret = 0;
cefc53c7	219
e5aa7f00 JR	220	if (unlikely(domain->ops->map == NULL))
e5aa7f00 JR	221	return -ENODEV;
cefc53c7	222
7d3002cc OBC	223	/* find out the minimum page size supported */
	224	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
	225
	226	/*
	227	* both the virtual address and the physical one, as well as
	228	* the size of the mapping, must be aligned (at least) to the
	229	* size of the smallest page supported by the hardware
	230	*/
	231	if (!IS_ALIGNED(iova \| paddr \| size, min_pagesz)) {
	232	pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz "
	233	"0x%x\n", iova, (unsigned long)paddr,
	234	(unsigned long)size, min_pagesz);
	235	return -EINVAL;
	236	}
	237
	238	pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova,
	239	(unsigned long)paddr, (unsigned long)size);
	240
	241	while (size) {
	242	unsigned long pgsize, addr_merge = iova \| paddr;
	243	unsigned int pgsize_idx;
	244
	245	/* Max page size that still fits into 'size' */
	246	pgsize_idx = __fls(size);
	247
	248	/* need to consider alignment requirements ? */
	249	if (likely(addr_merge)) {
	250	/* Max page size allowed by both iova and paddr */
	251	unsigned int align_pgsize_idx = __ffs(addr_merge);
	252
	253	pgsize_idx = min(pgsize_idx, align_pgsize_idx);
	254	}
	255
	256	/* build a mask of acceptable page sizes */
	257	pgsize = (1UL << (pgsize_idx + 1)) - 1;
	258
	259	/* throw away page sizes not supported by the hardware */
	260	pgsize &= domain->ops->pgsize_bitmap;
cefc53c7	261
7d3002cc OBC	262	/* make sure we're still sane */
7d3002cc OBC	263	BUG_ON(!pgsize);
cefc53c7	264
7d3002cc OBC	265	/* pick the biggest page */
	266	pgsize_idx = __fls(pgsize);
	267	pgsize = 1UL << pgsize_idx;
	268
	269	pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova,
	270	(unsigned long)paddr, pgsize);
	271
	272	ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
	273	if (ret)
	274	break;
	275
	276	iova += pgsize;
	277	paddr += pgsize;
	278	size -= pgsize;
	279	}
	280
	281	/* unroll mapping in case something went wrong */
	282	if (ret)
	283	iommu_unmap(domain, orig_iova, orig_size - size);
	284
	285	return ret;
cefc53c7 JR	286	}
	287	EXPORT_SYMBOL_GPL(iommu_map);
	288
7d3002cc	289	size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
cefc53c7	290	{
7d3002cc OBC	291	size_t unmapped_page, unmapped = 0;
7d3002cc OBC	292	unsigned int min_pagesz;
cefc53c7	293
e5aa7f00 JR	294	if (unlikely(domain->ops->unmap == NULL))
	295	return -ENODEV;
	296
7d3002cc OBC	297	/* find out the minimum page size supported */
	298	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
	299
	300	/*
	301	* The virtual address, as well as the size of the mapping, must be
	302	* aligned (at least) to the size of the smallest page supported
	303	* by the hardware
	304	*/
	305	if (!IS_ALIGNED(iova \| size, min_pagesz)) {
	306	pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n",
	307	iova, (unsigned long)size, min_pagesz);
	308	return -EINVAL;
	309	}
	310
	311	pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova,
	312	(unsigned long)size);
	313
	314	/*
	315	* Keep iterating until we either unmap 'size' bytes (or more)
	316	* or we hit an area that isn't mapped.
	317	*/
	318	while (unmapped < size) {
	319	size_t left = size - unmapped;
	320
	321	unmapped_page = domain->ops->unmap(domain, iova, left);
	322	if (!unmapped_page)
	323	break;
	324
	325	pr_debug("unmapped: iova 0x%lx size %lx\n", iova,
	326	(unsigned long)unmapped_page);
	327
	328	iova += unmapped_page;
	329	unmapped += unmapped_page;
	330	}
	331
	332	return unmapped;
cefc53c7 JR	333	}
cefc53c7 JR	334	EXPORT_SYMBOL_GPL(iommu_unmap);
1460432c AW	335
	336	int iommu_device_group(struct device dev, unsigned int groupid)
	337	{
	338	if (iommu_present(dev->bus) && dev->bus->iommu_ops->device_group)
	339	return dev->bus->iommu_ops->device_group(dev, groupid);
	340
	341	return -ENODEV;
	342	}
	343	EXPORT_SYMBOL_GPL(iommu_device_group);