[deliverable/linux.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dma-debug.h>
#include <linux/dmar.h>
#include <linux/bootmem.h>
#include <linux/pci.h>
#include <linux/kmemleak.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/iommu.h>
#include <asm/gart.h>
#include <asm/calgary.h>
#include <asm/amd_iommu.h>

static int forbid_dac __read_mostly;

struct dma_map_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/*
 * This variable becomes 1 if iommu=pt is passed on the kernel command line.
 * If this variable is 1, IOMMU implementations do no DMA translation for
 * devices and allow every device to access to whole physical memory. This is
 * useful if a user want to use an IOMMU only for KVM device assignment to
 * guests and not for driver dma translation.
 */
int iommu_pass_through __read_mostly;

dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);

/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to older i386. */
struct device x86_dma_fallback_dev = {
	.init_name = "fallback device",
	.coherent_dma_mask = DMA_BIT_MASK(32),
	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
};
EXPORT_SYMBOL(x86_dma_fallback_dev);

/* Number of entries preallocated for DMA-API debugging */
#define PREALLOC_DMA_DEBUG_ENTRIES       32768

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

#ifdef CONFIG_X86_64
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
	if (max_pfn <= MAX_DMA32_PFN)
		return;

	/*
	 * check aperture_64.c allocate_aperture() for reason about
	 * using 512M as goal
	 */
	align = 64ULL<<20;
	size = roundup(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
				 512ULL<<20);
	/*
	 * Kmemleak should not scan this block as it may not be mapped via the
	 * kernel direct mapping.
	 */
	kmemleak_ignore(dma32_bootmem_ptr);
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{

	if (max_pfn <= MAX_DMA32_PFN)
		return;

	if (!dma32_bootmem_ptr)
		return;

	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}
#endif

void __init pci_iommu_alloc(void)
{
	/* swiotlb is forced by the boot option */
	int use_swiotlb = swiotlb;
#ifdef CONFIG_X86_64
	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();
#else
	dma_ops = &nommu_dma_ops;
#endif
	pci_swiotlb_init();
	if (use_swiotlb)
		return;

	gart_iommu_hole_init();

	detect_calgary();

	detect_intel_iommu();

	/* needs to be called after gart_iommu_hole_init */
	amd_iommu_detect();
}

void *dma_generic_alloc_coherent(struct device *dev, size_t size,
				 dma_addr_t *dma_addr, gfp_t flag)
{
	unsigned long dma_mask;
	struct page *page;
	dma_addr_t addr;

	dma_mask = dma_alloc_coherent_mask(dev, flag);

	flag |= __GFP_ZERO;
again:
	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	if (!page)
		return NULL;

	addr = page_to_phys(page);
	if (addr + size > dma_mask) {
		__free_pages(page, get_order(size));

		if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
			flag = (flag & ~GFP_DMA32) | GFP_DMA;
			goto again;
		}

		return NULL;
	}

	*dma_addr = addr;
	return page_address(page);
}

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = 1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif
		if (!strncmp(p, "pt", 2))
			iommu_pass_through = 1;

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

int dma_supported(struct device *dev, u64 mask)
{
	struct dma_map_ops *ops = get_dma_ops(dev);

#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		dev_info(dev, "PCI: Disallowing DAC for device\n");
		return 0;
	}
#endif

	if (ops->dma_supported)
		return ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_BIT_MASK(24))
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

static int __init pci_iommu_init(void)
{
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

#ifdef CONFIG_PCI
	dma_debug_add_bus(&pci_bus_type);
#endif
	x86_init.iommu.iommu_init();

	if (swiotlb) {
		printk(KERN_INFO "PCI-DMA: "
		       "Using software bounce buffering for IO (SWIOTLB)\n");
		swiotlb_print_info();
	} else
		swiotlb_free();

	return 0;
}
/* Must execute after PCI subsystem */
rootfs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
2118d0c5	2	#include <linux/dma-debug.h>
cb5867a5	3	#include <linux/dmar.h>
116890d5	4	#include <linux/bootmem.h>
bca5c096	5	#include <linux/pci.h>
acde31dc	6	#include <linux/kmemleak.h>
cb5867a5	7
116890d5 GC	8	#include <asm/proto.h>
116890d5 GC	9	#include <asm/dma.h>
46a7fa27	10	#include <asm/iommu.h>
1d9b16d1	11	#include <asm/gart.h>
cb5867a5	12	#include <asm/calgary.h>
a69ca340	13	#include <asm/amd_iommu.h>
459121c9	14
3b15e581 FY	15	static int forbid_dac __read_mostly;
3b15e581 FY	16
160c1d8e	17	struct dma_map_ops *dma_ops;
85c246ee GC	18	EXPORT_SYMBOL(dma_ops);
85c246ee GC	19
b4cdc430	20	static int iommu_sac_force __read_mostly;
8e0c3797	21
f9c258de GC	22	#ifdef CONFIG_IOMMU_DEBUG
	23	int panic_on_overflow __read_mostly = 1;
	24	int force_iommu __read_mostly = 1;
	25	#else
	26	int panic_on_overflow __read_mostly = 0;
	27	int force_iommu __read_mostly = 0;
	28	#endif
	29
fae9a0d8 GC	30	int iommu_merge __read_mostly = 0;
	31
	32	int no_iommu __read_mostly;
	33	/* Set this to 1 if there is a HW IOMMU in the system */
	34	int iommu_detected __read_mostly = 0;
	35
ac0101d3 JR	36	/*
ac0101d3 JR	37	* This variable becomes 1 if iommu=pt is passed on the kernel command line.
e3be785f	38	* If this variable is 1, IOMMU implementations do no DMA translation for
ac0101d3 JR	39	* devices and allow every device to access to whole physical memory. This is
	40	* useful if a user want to use an IOMMU only for KVM device assignment to
	41	* guests and not for driver dma translation.
	42	*/
	43	int iommu_pass_through __read_mostly;
aed5d5f4	44
cac67877 GC	45	dma_addr_t bad_dma_address __read_mostly = 0;
cac67877 GC	46	EXPORT_SYMBOL(bad_dma_address);
fae9a0d8	47
098cb7f2 GC	48	/* Dummy device used for NULL arguments (normally ISA). Better would
	49	be probably a smaller DMA mask, but this is bug-to-bug compatible
	50	to older i386. */
6c505ce3	51	struct device x86_dma_fallback_dev = {
1a927133	52	.init_name = "fallback device",
284901a9	53	.coherent_dma_mask = DMA_BIT_MASK(32),
6c505ce3	54	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
098cb7f2	55	};
6c505ce3	56	EXPORT_SYMBOL(x86_dma_fallback_dev);
098cb7f2	57
2118d0c5 JR	58	/* Number of entries preallocated for DMA-API debugging */
	59	#define PREALLOC_DMA_DEBUG_ENTRIES 32768
	60
459121c9 GC	61	int dma_set_mask(struct device *dev, u64 mask)
	62	{
	63	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	64	return -EIO;
	65
	66	*dev->dma_mask = mask;
	67
	68	return 0;
	69	}
	70	EXPORT_SYMBOL(dma_set_mask);
	71
116890d5 GC	72	#ifdef CONFIG_X86_64
	73	static __initdata void *dma32_bootmem_ptr;
	74	static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
	75
	76	static int __init parse_dma32_size_opt(char *p)
	77	{
	78	if (!p)
	79	return -EINVAL;
	80	dma32_bootmem_size = memparse(p, &p);
	81	return 0;
	82	}
	83	early_param("dma32_size", parse_dma32_size_opt);
	84
	85	void __init dma32_reserve_bootmem(void)
	86	{
	87	unsigned long size, align;
c987d12f	88	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	89	return;
116890d5 GC	90
7677b2ef YL	91	/*
	92	* check aperture_64.c allocate_aperture() for reason about
	93	* using 512M as goal
	94	*/
116890d5	95	align = 64ULL<<20;
1ddb5518	96	size = roundup(dma32_bootmem_size, align);
116890d5	97	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
7677b2ef	98	512ULL<<20);
acde31dc CM	99	/*
	100	* Kmemleak should not scan this block as it may not be mapped via the
	101	* kernel direct mapping.
	102	*/
	103	kmemleak_ignore(dma32_bootmem_ptr);
116890d5 GC	104	if (dma32_bootmem_ptr)
	105	dma32_bootmem_size = size;
	106	else
	107	dma32_bootmem_size = 0;
	108	}
	109	static void __init dma32_free_bootmem(void)
	110	{
116890d5	111
c987d12f	112	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	113	return;
	114
	115	if (!dma32_bootmem_ptr)
	116	return;
	117
330fce23	118	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
116890d5 GC	119
	120	dma32_bootmem_ptr = NULL;
	121	dma32_bootmem_size = 0;
	122	}
cfb80c9e	123	#endif
116890d5 GC	124
	125	void __init pci_iommu_alloc(void)
	126	{
75f1cdf1 FT	127	/* swiotlb is forced by the boot option */
75f1cdf1 FT	128	int use_swiotlb = swiotlb;
cfb80c9e	129	#ifdef CONFIG_X86_64
116890d5 GC	130	/* free the range so iommu could get some range less than 4G */
116890d5 GC	131	dma32_free_bootmem();
72d03802 FT	132	#else
72d03802 FT	133	dma_ops = &nommu_dma_ops;
cfb80c9e	134	#endif
75f1cdf1 FT	135	pci_swiotlb_init();
	136	if (use_swiotlb)
	137	return;
cfb80c9e	138
116890d5	139	gart_iommu_hole_init();
116890d5	140
116890d5	141	detect_calgary();
116890d5 GC	142
	143	detect_intel_iommu();
	144
75f1cdf1	145	/* needs to be called after gart_iommu_hole_init */
a69ca340	146	amd_iommu_detect();
116890d5	147	}
8978b742	148
9f6ac577 FT	149	void dma_generic_alloc_coherent(struct device dev, size_t size,
	150	dma_addr_t *dma_addr, gfp_t flag)
	151	{
	152	unsigned long dma_mask;
	153	struct page *page;
	154	dma_addr_t addr;
	155
	156	dma_mask = dma_alloc_coherent_mask(dev, flag);
	157
	158	flag \|= __GFP_ZERO;
	159	again:
	160	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	161	if (!page)
	162	return NULL;
	163
	164	addr = page_to_phys(page);
a4c2baa6	165	if (addr + size > dma_mask) {
9f6ac577 FT	166	__free_pages(page, get_order(size));
9f6ac577 FT	167
284901a9	168	if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
9f6ac577 FT	169	flag = (flag & ~GFP_DMA32) \| GFP_DMA;
	170	goto again;
	171	}
	172
	173	return NULL;
	174	}
	175
	176	*dma_addr = addr;
	177	return page_address(page);
	178	}
	179
fae9a0d8 GC	180	/*
	181	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	182	* documentation.
	183	*/
	184	static __init int iommu_setup(char *p)
	185	{
	186	iommu_merge = 1;
	187
	188	if (!p)
	189	return -EINVAL;
	190
	191	while (*p) {
	192	if (!strncmp(p, "off", 3))
	193	no_iommu = 1;
	194	/* gart_parse_options has more force support */
	195	if (!strncmp(p, "force", 5))
	196	force_iommu = 1;
	197	if (!strncmp(p, "noforce", 7)) {
	198	iommu_merge = 0;
	199	force_iommu = 0;
	200	}
	201
	202	if (!strncmp(p, "biomerge", 8)) {
fae9a0d8 GC	203	iommu_merge = 1;
	204	force_iommu = 1;
	205	}
	206	if (!strncmp(p, "panic", 5))
	207	panic_on_overflow = 1;
	208	if (!strncmp(p, "nopanic", 7))
	209	panic_on_overflow = 0;
	210	if (!strncmp(p, "merge", 5)) {
	211	iommu_merge = 1;
	212	force_iommu = 1;
	213	}
	214	if (!strncmp(p, "nomerge", 7))
	215	iommu_merge = 0;
	216	if (!strncmp(p, "forcesac", 8))
	217	iommu_sac_force = 1;
	218	if (!strncmp(p, "allowdac", 8))
	219	forbid_dac = 0;
	220	if (!strncmp(p, "nodac", 5))
2ae8bb75	221	forbid_dac = 1;
fae9a0d8 GC	222	if (!strncmp(p, "usedac", 6)) {
	223	forbid_dac = -1;
	224	return 1;
	225	}
	226	#ifdef CONFIG_SWIOTLB
	227	if (!strncmp(p, "soft", 4))
	228	swiotlb = 1;
3238c0c4	229	#endif
80286879	230	if (!strncmp(p, "pt", 2))
4ed0d3e6	231	iommu_pass_through = 1;
fae9a0d8	232
fae9a0d8	233	gart_parse_options(p);
fae9a0d8 GC	234
	235	#ifdef CONFIG_CALGARY_IOMMU
	236	if (!strncmp(p, "calgary", 7))
	237	use_calgary = 1;
	238	#endif /* CONFIG_CALGARY_IOMMU */
	239
	240	p += strcspn(p, ",");
	241	if (*p == ',')
	242	++p;
	243	}
	244	return 0;
	245	}
	246	early_param("iommu", iommu_setup);
	247
8e0c3797 GC	248	int dma_supported(struct device *dev, u64 mask)
8e0c3797 GC	249	{
160c1d8e	250	struct dma_map_ops *ops = get_dma_ops(dev);
8d8bb39b	251
8e0c3797 GC	252	#ifdef CONFIG_PCI
8e0c3797 GC	253	if (mask > 0xffffffff && forbid_dac > 0) {
fc3a8828	254	dev_info(dev, "PCI: Disallowing DAC for device\n");
8e0c3797 GC	255	return 0;
	256	}
	257	#endif
	258
8d8bb39b FT	259	if (ops->dma_supported)
8d8bb39b FT	260	return ops->dma_supported(dev, mask);
8e0c3797 GC	261
	262	/* Copied from i386. Doesn't make much sense, because it will
	263	only work for pci_alloc_coherent.
	264	The caller just has to use GFP_DMA in this case. */
2f4f27d4	265	if (mask < DMA_BIT_MASK(24))
8e0c3797 GC	266	return 0;
	267
	268	/* Tell the device to use SAC when IOMMU force is on. This
	269	allows the driver to use cheaper accesses in some cases.
	270
	271	Problem with this is that if we overflow the IOMMU area and
	272	return DAC as fallback address the device may not handle it
	273	correctly.
	274
	275	As a special case some controllers have a 39bit address
	276	mode that is as efficient as 32bit (aic79xx). Don't force
	277	SAC for these. Assume all masks <= 40 bits are of this
	278	type. Normally this doesn't make any difference, but gives
	279	more gentle handling of IOMMU overflow. */
50cf156a	280	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
fc3a8828	281	dev_info(dev, "Force SAC with mask %Lx\n", mask);
8e0c3797 GC	282	return 0;
	283	}
	284
	285	return 1;
	286	}
	287	EXPORT_SYMBOL(dma_supported);
	288
cb5867a5 GC	289	static int __init pci_iommu_init(void)
cb5867a5 GC	290	{
2118d0c5 JR	291	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
2118d0c5 JR	292
86f31952 JR	293	#ifdef CONFIG_PCI
	294	dma_debug_add_bus(&pci_bus_type);
	295	#endif
d07c1be0 FT	296	x86_init.iommu.iommu_init();
d07c1be0 FT	297
75f1cdf1 FT	298	if (swiotlb) {
	299	printk(KERN_INFO "PCI-DMA: "
	300	"Using software bounce buffering for IO (SWIOTLB)\n");
	301	swiotlb_print_info();
	302	} else
	303	swiotlb_free();
	304
cb5867a5 GC	305	return 0;
cb5867a5 GC	306	}
cb5867a5	307	/* Must execute after PCI subsystem */
9a821b23	308	rootfs_initcall(pci_iommu_init);
3b15e581 FY	309
	310	#ifdef CONFIG_PCI
	311	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	312
	313	static __devinit void via_no_dac(struct pci_dev *dev)
	314	{
	315	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
13bf7576	316	dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
3b15e581 FY	317	forbid_dac = 1;
	318	}
	319	}
	320	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	321	#endif