[deliverable/linux.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dma-debug.h>
#include <linux/dmar.h>
#include <linux/export.h>
#include <linux/bootmem.h>
#include <linux/gfp.h>
#include <linux/pci.h>
#include <linux/kmemleak.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/iommu.h>
#include <asm/gart.h>
#include <asm/calgary.h>
#include <asm/x86_init.h>
#include <asm/iommu_table.h>

static int forbid_dac __read_mostly;

struct dma_map_ops *dma_ops = &nommu_dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/*
 * This variable becomes 1 if iommu=pt is passed on the kernel command line.
 * If this variable is 1, IOMMU implementations do no DMA translation for
 * devices and allow every device to access to whole physical memory. This is
 * useful if a user wants to use an IOMMU only for KVM device assignment to
 * guests and not for driver dma translation.
 */
int iommu_pass_through __read_mostly;

/*
 * Group multi-function PCI devices into a single device-group for the
 * iommu_device_group interface.  This tells the iommu driver to pretend
 * it cannot distinguish between functions of a device, exposing only one
 * group for the device.  Useful for disallowing use of individual PCI
 * functions from userspace drivers.
 */
int iommu_group_mf __read_mostly;

extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];

/* Dummy device used for NULL arguments (normally ISA). */
struct device x86_dma_fallback_dev = {
	.init_name = "fallback device",
	.coherent_dma_mask = ISA_DMA_BIT_MASK,
	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
};
EXPORT_SYMBOL(x86_dma_fallback_dev);

/* Number of entries preallocated for DMA-API debugging */
#define PREALLOC_DMA_DEBUG_ENTRIES       32768

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

void __init pci_iommu_alloc(void)
{
	struct iommu_table_entry *p;

	sort_iommu_table(__iommu_table, __iommu_table_end);
	check_iommu_entries(__iommu_table, __iommu_table_end);

	for (p = __iommu_table; p < __iommu_table_end; p++) {
		if (p && p->detect && p->detect() > 0) {
			p->flags |= IOMMU_DETECTED;
			if (p->early_init)
				p->early_init();
			if (p->flags & IOMMU_FINISH_IF_DETECTED)
				break;
		}
	}
}
void *dma_generic_alloc_coherent(struct device *dev, size_t size,
				 dma_addr_t *dma_addr, gfp_t flag)
{
	unsigned long dma_mask;
	struct page *page;
	dma_addr_t addr;

	dma_mask = dma_alloc_coherent_mask(dev, flag);

	flag |= __GFP_ZERO;
again:
	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	if (!page)
		return NULL;

	addr = page_to_phys(page);
	if (addr + size > dma_mask) {
		__free_pages(page, get_order(size));

		if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
			flag = (flag & ~GFP_DMA32) | GFP_DMA;
			goto again;
		}

		return NULL;
	}

	*dma_addr = addr;
	return page_address(page);
}

/*
 * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
 * parameter documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = 1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif
		if (!strncmp(p, "pt", 2))
			iommu_pass_through = 1;
		if (!strncmp(p, "group_mf", 8))
			iommu_group_mf = 1;

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

int dma_supported(struct device *dev, u64 mask)
{
	struct dma_map_ops *ops = get_dma_ops(dev);

#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		dev_info(dev, "PCI: Disallowing DAC for device\n");
		return 0;
	}
#endif

	if (ops->dma_supported)
		return ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_BIT_MASK(24))
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

static int __init pci_iommu_init(void)
{
	struct iommu_table_entry *p;
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

#ifdef CONFIG_PCI
	dma_debug_add_bus(&pci_bus_type);
#endif
	x86_init.iommu.iommu_init();

	for (p = __iommu_table; p < __iommu_table_end; p++) {
		if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
			p->late_init();
	}

	return 0;
}
/* Must execute after PCI subsystem */
rootfs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
2118d0c5	2	#include <linux/dma-debug.h>
cb5867a5	3	#include <linux/dmar.h>
69c60c88	4	#include <linux/export.h>
116890d5	5	#include <linux/bootmem.h>
5a0e3ad6	6	#include <linux/gfp.h>
bca5c096	7	#include <linux/pci.h>
acde31dc	8	#include <linux/kmemleak.h>
cb5867a5	9
116890d5 GC	10	#include <asm/proto.h>
116890d5 GC	11	#include <asm/dma.h>
46a7fa27	12	#include <asm/iommu.h>
1d9b16d1	13	#include <asm/gart.h>
cb5867a5	14	#include <asm/calgary.h>
b4941a9a	15	#include <asm/x86_init.h>
ee1f284f	16	#include <asm/iommu_table.h>
459121c9	17
3b15e581 FY	18	static int forbid_dac __read_mostly;
3b15e581 FY	19
a3b28ee1	20	struct dma_map_ops *dma_ops = &nommu_dma_ops;
85c246ee GC	21	EXPORT_SYMBOL(dma_ops);
85c246ee GC	22
b4cdc430	23	static int iommu_sac_force __read_mostly;
8e0c3797	24
f9c258de GC	25	#ifdef CONFIG_IOMMU_DEBUG
	26	int panic_on_overflow __read_mostly = 1;
	27	int force_iommu __read_mostly = 1;
	28	#else
	29	int panic_on_overflow __read_mostly = 0;
	30	int force_iommu __read_mostly = 0;
	31	#endif
	32
fae9a0d8 GC	33	int iommu_merge __read_mostly = 0;
	34
	35	int no_iommu __read_mostly;
	36	/* Set this to 1 if there is a HW IOMMU in the system */
	37	int iommu_detected __read_mostly = 0;
	38
ac0101d3 JR	39	/*
ac0101d3 JR	40	* This variable becomes 1 if iommu=pt is passed on the kernel command line.
e3be785f	41	* If this variable is 1, IOMMU implementations do no DMA translation for
ac0101d3	42	* devices and allow every device to access to whole physical memory. This is
fb637f3c	43	* useful if a user wants to use an IOMMU only for KVM device assignment to
ac0101d3 JR	44	* guests and not for driver dma translation.
	45	*/
	46	int iommu_pass_through __read_mostly;
aed5d5f4	47
bcb71abe AW	48	/*
	49	* Group multi-function PCI devices into a single device-group for the
	50	* iommu_device_group interface. This tells the iommu driver to pretend
	51	* it cannot distinguish between functions of a device, exposing only one
	52	* group for the device. Useful for disallowing use of individual PCI
	53	* functions from userspace drivers.
	54	*/
	55	int iommu_group_mf __read_mostly;
	56
ee1f284f KRW	57	extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
ee1f284f KRW	58
eb647138	59	/* Dummy device used for NULL arguments (normally ISA). */
6c505ce3	60	struct device x86_dma_fallback_dev = {
1a927133	61	.init_name = "fallback device",
eb647138	62	.coherent_dma_mask = ISA_DMA_BIT_MASK,
6c505ce3	63	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
098cb7f2	64	};
6c505ce3	65	EXPORT_SYMBOL(x86_dma_fallback_dev);
098cb7f2	66
2118d0c5 JR	67	/* Number of entries preallocated for DMA-API debugging */
	68	#define PREALLOC_DMA_DEBUG_ENTRIES 32768
	69
459121c9 GC	70	int dma_set_mask(struct device *dev, u64 mask)
	71	{
	72	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	73	return -EIO;
	74
	75	*dev->dma_mask = mask;
	76
	77	return 0;
	78	}
	79	EXPORT_SYMBOL(dma_set_mask);
	80
116890d5 GC	81	void __init pci_iommu_alloc(void)
116890d5 GC	82	{
ee1f284f KRW	83	struct iommu_table_entry *p;
ee1f284f KRW	84
ee1f284f KRW	85	sort_iommu_table(__iommu_table, __iommu_table_end);
ee1f284f KRW	86	check_iommu_entries(__iommu_table, __iommu_table_end);
116890d5	87
ee1f284f KRW	88	for (p = __iommu_table; p < __iommu_table_end; p++) {
	89	if (p && p->detect && p->detect() > 0) {
	90	p->flags \|= IOMMU_DETECTED;
	91	if (p->early_init)
	92	p->early_init();
	93	if (p->flags & IOMMU_FINISH_IF_DETECTED)
	94	break;
	95	}
	96	}
116890d5	97	}
9f6ac577 FT	98	void dma_generic_alloc_coherent(struct device dev, size_t size,
	99	dma_addr_t *dma_addr, gfp_t flag)
	100	{
	101	unsigned long dma_mask;
	102	struct page *page;
	103	dma_addr_t addr;
	104
	105	dma_mask = dma_alloc_coherent_mask(dev, flag);
	106
	107	flag \|= __GFP_ZERO;
	108	again:
	109	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	110	if (!page)
	111	return NULL;
	112
	113	addr = page_to_phys(page);
a4c2baa6	114	if (addr + size > dma_mask) {
9f6ac577 FT	115	__free_pages(page, get_order(size));
9f6ac577 FT	116
284901a9	117	if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
9f6ac577 FT	118	flag = (flag & ~GFP_DMA32) \| GFP_DMA;
	119	goto again;
	120	}
	121
	122	return NULL;
	123	}
	124
	125	*dma_addr = addr;
	126	return page_address(page);
	127	}
	128
fae9a0d8	129	/*
395cf969 PB	130	* See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
395cf969 PB	131	* parameter documentation.
fae9a0d8 GC	132	*/
	133	static __init int iommu_setup(char *p)
	134	{
	135	iommu_merge = 1;
	136
	137	if (!p)
	138	return -EINVAL;
	139
	140	while (*p) {
	141	if (!strncmp(p, "off", 3))
	142	no_iommu = 1;
	143	/* gart_parse_options has more force support */
	144	if (!strncmp(p, "force", 5))
	145	force_iommu = 1;
	146	if (!strncmp(p, "noforce", 7)) {
	147	iommu_merge = 0;
	148	force_iommu = 0;
	149	}
	150
	151	if (!strncmp(p, "biomerge", 8)) {
fae9a0d8 GC	152	iommu_merge = 1;
	153	force_iommu = 1;
	154	}
	155	if (!strncmp(p, "panic", 5))
	156	panic_on_overflow = 1;
	157	if (!strncmp(p, "nopanic", 7))
	158	panic_on_overflow = 0;
	159	if (!strncmp(p, "merge", 5)) {
	160	iommu_merge = 1;
	161	force_iommu = 1;
	162	}
	163	if (!strncmp(p, "nomerge", 7))
	164	iommu_merge = 0;
	165	if (!strncmp(p, "forcesac", 8))
	166	iommu_sac_force = 1;
	167	if (!strncmp(p, "allowdac", 8))
	168	forbid_dac = 0;
	169	if (!strncmp(p, "nodac", 5))
2ae8bb75	170	forbid_dac = 1;
fae9a0d8 GC	171	if (!strncmp(p, "usedac", 6)) {
	172	forbid_dac = -1;
	173	return 1;
	174	}
	175	#ifdef CONFIG_SWIOTLB
	176	if (!strncmp(p, "soft", 4))
	177	swiotlb = 1;
3238c0c4	178	#endif
80286879	179	if (!strncmp(p, "pt", 2))
4ed0d3e6	180	iommu_pass_through = 1;
bcb71abe AW	181	if (!strncmp(p, "group_mf", 8))
bcb71abe AW	182	iommu_group_mf = 1;
fae9a0d8	183
fae9a0d8	184	gart_parse_options(p);
fae9a0d8 GC	185
	186	#ifdef CONFIG_CALGARY_IOMMU
	187	if (!strncmp(p, "calgary", 7))
	188	use_calgary = 1;
	189	#endif /* CONFIG_CALGARY_IOMMU */
	190
	191	p += strcspn(p, ",");
	192	if (*p == ',')
	193	++p;
	194	}
	195	return 0;
	196	}
	197	early_param("iommu", iommu_setup);
	198
8e0c3797 GC	199	int dma_supported(struct device *dev, u64 mask)
8e0c3797 GC	200	{
160c1d8e	201	struct dma_map_ops *ops = get_dma_ops(dev);
8d8bb39b	202
8e0c3797 GC	203	#ifdef CONFIG_PCI
8e0c3797 GC	204	if (mask > 0xffffffff && forbid_dac > 0) {
fc3a8828	205	dev_info(dev, "PCI: Disallowing DAC for device\n");
8e0c3797 GC	206	return 0;
	207	}
	208	#endif
	209
8d8bb39b FT	210	if (ops->dma_supported)
8d8bb39b FT	211	return ops->dma_supported(dev, mask);
8e0c3797 GC	212
	213	/* Copied from i386. Doesn't make much sense, because it will
	214	only work for pci_alloc_coherent.
	215	The caller just has to use GFP_DMA in this case. */
2f4f27d4	216	if (mask < DMA_BIT_MASK(24))
8e0c3797 GC	217	return 0;
	218
	219	/* Tell the device to use SAC when IOMMU force is on. This
	220	allows the driver to use cheaper accesses in some cases.
	221
	222	Problem with this is that if we overflow the IOMMU area and
	223	return DAC as fallback address the device may not handle it
	224	correctly.
	225
	226	As a special case some controllers have a 39bit address
	227	mode that is as efficient as 32bit (aic79xx). Don't force
	228	SAC for these. Assume all masks <= 40 bits are of this
	229	type. Normally this doesn't make any difference, but gives
	230	more gentle handling of IOMMU overflow. */
50cf156a	231	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
fc3a8828	232	dev_info(dev, "Force SAC with mask %Lx\n", mask);
8e0c3797 GC	233	return 0;
	234	}
	235
	236	return 1;
	237	}
	238	EXPORT_SYMBOL(dma_supported);
	239
cb5867a5 GC	240	static int __init pci_iommu_init(void)
cb5867a5 GC	241	{
ee1f284f	242	struct iommu_table_entry *p;
2118d0c5 JR	243	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
2118d0c5 JR	244
86f31952 JR	245	#ifdef CONFIG_PCI
	246	dma_debug_add_bus(&pci_bus_type);
	247	#endif
d07c1be0 FT	248	x86_init.iommu.iommu_init();
d07c1be0 FT	249
ee1f284f KRW	250	for (p = __iommu_table; p < __iommu_table_end; p++) {
	251	if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
	252	p->late_init();
	253	}
75f1cdf1	254
cb5867a5 GC	255	return 0;
cb5867a5 GC	256	}
cb5867a5	257	/* Must execute after PCI subsystem */
9a821b23	258	rootfs_initcall(pci_iommu_init);
3b15e581 FY	259
	260	#ifdef CONFIG_PCI
	261	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	262
	263	static __devinit void via_no_dac(struct pci_dev *dev)
	264	{
	265	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
13bf7576	266	dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
3b15e581 FY	267	forbid_dac = 1;
	268	}
	269	}
	270	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	271	#endif