[deliverable/linux.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dma-debug.h>
#include <linux/dmar.h>
#include <linux/bootmem.h>
#include <linux/gfp.h>
#include <linux/pci.h>
#include <linux/kmemleak.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/iommu.h>
#include <asm/gart.h>
#include <asm/calgary.h>
#include <asm/x86_init.h>
#include <asm/iommu_table.h>

static int forbid_dac __read_mostly;

struct dma_map_ops *dma_ops = &nommu_dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/*
 * This variable becomes 1 if iommu=pt is passed on the kernel command line.
 * If this variable is 1, IOMMU implementations do no DMA translation for
 * devices and allow every device to access to whole physical memory. This is
 * useful if a user wants to use an IOMMU only for KVM device assignment to
 * guests and not for driver dma translation.
 */
int iommu_pass_through __read_mostly;

extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];

/* Dummy device used for NULL arguments (normally ISA). */
struct device x86_dma_fallback_dev = {
	.init_name = "fallback device",
	.coherent_dma_mask = ISA_DMA_BIT_MASK,
	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
};
EXPORT_SYMBOL(x86_dma_fallback_dev);

/* Number of entries preallocated for DMA-API debugging */
#define PREALLOC_DMA_DEBUG_ENTRIES       32768

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

#if defined(CONFIG_X86_64) && !defined(CONFIG_NUMA)
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
	if (max_pfn <= MAX_DMA32_PFN)
		return;

	/*
	 * check aperture_64.c allocate_aperture() for reason about
	 * using 512M as goal
	 */
	align = 64ULL<<20;
	size = roundup(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
				 512ULL<<20);
	/*
	 * Kmemleak should not scan this block as it may not be mapped via the
	 * kernel direct mapping.
	 */
	kmemleak_ignore(dma32_bootmem_ptr);
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{

	if (max_pfn <= MAX_DMA32_PFN)
		return;

	if (!dma32_bootmem_ptr)
		return;

	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}
#else
void __init dma32_reserve_bootmem(void)
{
}
static void __init dma32_free_bootmem(void)
{
}

#endif

void __init pci_iommu_alloc(void)
{
	struct iommu_table_entry *p;

	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();

	sort_iommu_table(__iommu_table, __iommu_table_end);
	check_iommu_entries(__iommu_table, __iommu_table_end);

	for (p = __iommu_table; p < __iommu_table_end; p++) {
		if (p && p->detect && p->detect() > 0) {
			p->flags |= IOMMU_DETECTED;
			if (p->early_init)
				p->early_init();
			if (p->flags & IOMMU_FINISH_IF_DETECTED)
				break;
		}
	}
}
void *dma_generic_alloc_coherent(struct device *dev, size_t size,
				 dma_addr_t *dma_addr, gfp_t flag)
{
	unsigned long dma_mask;
	struct page *page;
	dma_addr_t addr;

	dma_mask = dma_alloc_coherent_mask(dev, flag);

	flag |= __GFP_ZERO;
again:
	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	if (!page)
		return NULL;

	addr = page_to_phys(page);
	if (addr + size > dma_mask) {
		__free_pages(page, get_order(size));

		if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
			flag = (flag & ~GFP_DMA32) | GFP_DMA;
			goto again;
		}

		return NULL;
	}

	*dma_addr = addr;
	return page_address(page);
}

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = 1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif
		if (!strncmp(p, "pt", 2))
			iommu_pass_through = 1;

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

int dma_supported(struct device *dev, u64 mask)
{
	struct dma_map_ops *ops = get_dma_ops(dev);

#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		dev_info(dev, "PCI: Disallowing DAC for device\n");
		return 0;
	}
#endif

	if (ops->dma_supported)
		return ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_BIT_MASK(24))
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

static int __init pci_iommu_init(void)
{
	struct iommu_table_entry *p;
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

#ifdef CONFIG_PCI
	dma_debug_add_bus(&pci_bus_type);
#endif
	x86_init.iommu.iommu_init();

	for (p = __iommu_table; p < __iommu_table_end; p++) {
		if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
			p->late_init();
	}

	return 0;
}
/* Must execute after PCI subsystem */
rootfs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
2118d0c5	2	#include <linux/dma-debug.h>
cb5867a5	3	#include <linux/dmar.h>
116890d5	4	#include <linux/bootmem.h>
5a0e3ad6	5	#include <linux/gfp.h>
bca5c096	6	#include <linux/pci.h>
acde31dc	7	#include <linux/kmemleak.h>
cb5867a5	8
116890d5 GC	9	#include <asm/proto.h>
116890d5 GC	10	#include <asm/dma.h>
46a7fa27	11	#include <asm/iommu.h>
1d9b16d1	12	#include <asm/gart.h>
cb5867a5	13	#include <asm/calgary.h>
b4941a9a	14	#include <asm/x86_init.h>
ee1f284f	15	#include <asm/iommu_table.h>
459121c9	16
3b15e581 FY	17	static int forbid_dac __read_mostly;
3b15e581 FY	18
a3b28ee1	19	struct dma_map_ops *dma_ops = &nommu_dma_ops;
85c246ee GC	20	EXPORT_SYMBOL(dma_ops);
85c246ee GC	21
b4cdc430	22	static int iommu_sac_force __read_mostly;
8e0c3797	23
f9c258de GC	24	#ifdef CONFIG_IOMMU_DEBUG
	25	int panic_on_overflow __read_mostly = 1;
	26	int force_iommu __read_mostly = 1;
	27	#else
	28	int panic_on_overflow __read_mostly = 0;
	29	int force_iommu __read_mostly = 0;
	30	#endif
	31
fae9a0d8 GC	32	int iommu_merge __read_mostly = 0;
	33
	34	int no_iommu __read_mostly;
	35	/* Set this to 1 if there is a HW IOMMU in the system */
	36	int iommu_detected __read_mostly = 0;
	37
ac0101d3 JR	38	/*
ac0101d3 JR	39	* This variable becomes 1 if iommu=pt is passed on the kernel command line.
e3be785f	40	* If this variable is 1, IOMMU implementations do no DMA translation for
ac0101d3	41	* devices and allow every device to access to whole physical memory. This is
fb637f3c	42	* useful if a user wants to use an IOMMU only for KVM device assignment to
ac0101d3 JR	43	* guests and not for driver dma translation.
	44	*/
	45	int iommu_pass_through __read_mostly;
aed5d5f4	46
ee1f284f KRW	47	extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
ee1f284f KRW	48
eb647138	49	/* Dummy device used for NULL arguments (normally ISA). */
6c505ce3	50	struct device x86_dma_fallback_dev = {
1a927133	51	.init_name = "fallback device",
eb647138	52	.coherent_dma_mask = ISA_DMA_BIT_MASK,
6c505ce3	53	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
098cb7f2	54	};
6c505ce3	55	EXPORT_SYMBOL(x86_dma_fallback_dev);
098cb7f2	56
2118d0c5 JR	57	/* Number of entries preallocated for DMA-API debugging */
	58	#define PREALLOC_DMA_DEBUG_ENTRIES 32768
	59
459121c9 GC	60	int dma_set_mask(struct device *dev, u64 mask)
	61	{
	62	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	63	return -EIO;
	64
	65	*dev->dma_mask = mask;
	66
	67	return 0;
	68	}
	69	EXPORT_SYMBOL(dma_set_mask);
	70
c252a5bb	71	#if defined(CONFIG_X86_64) && !defined(CONFIG_NUMA)
116890d5 GC	72	static __initdata void *dma32_bootmem_ptr;
	73	static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
	74
	75	static int __init parse_dma32_size_opt(char *p)
	76	{
	77	if (!p)
	78	return -EINVAL;
	79	dma32_bootmem_size = memparse(p, &p);
	80	return 0;
	81	}
	82	early_param("dma32_size", parse_dma32_size_opt);
	83
	84	void __init dma32_reserve_bootmem(void)
	85	{
	86	unsigned long size, align;
c987d12f	87	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	88	return;
116890d5 GC	89
7677b2ef YL	90	/*
	91	* check aperture_64.c allocate_aperture() for reason about
	92	* using 512M as goal
	93	*/
116890d5	94	align = 64ULL<<20;
1ddb5518	95	size = roundup(dma32_bootmem_size, align);
116890d5	96	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
7677b2ef	97	512ULL<<20);
acde31dc CM	98	/*
	99	* Kmemleak should not scan this block as it may not be mapped via the
	100	* kernel direct mapping.
	101	*/
	102	kmemleak_ignore(dma32_bootmem_ptr);
116890d5 GC	103	if (dma32_bootmem_ptr)
	104	dma32_bootmem_size = size;
	105	else
	106	dma32_bootmem_size = 0;
	107	}
	108	static void __init dma32_free_bootmem(void)
	109	{
116890d5	110
c987d12f	111	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	112	return;
	113
	114	if (!dma32_bootmem_ptr)
	115	return;
	116
330fce23	117	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
116890d5 GC	118
	119	dma32_bootmem_ptr = NULL;
	120	dma32_bootmem_size = 0;
	121	}
c252a5bb YL	122	#else
	123	void __init dma32_reserve_bootmem(void)
	124	{
	125	}
	126	static void __init dma32_free_bootmem(void)
	127	{
	128	}
	129
cfb80c9e	130	#endif
116890d5 GC	131
	132	void __init pci_iommu_alloc(void)
	133	{
ee1f284f KRW	134	struct iommu_table_entry *p;
ee1f284f KRW	135
116890d5 GC	136	/* free the range so iommu could get some range less than 4G */
116890d5 GC	137	dma32_free_bootmem();
c252a5bb	138
ee1f284f KRW	139	sort_iommu_table(__iommu_table, __iommu_table_end);
ee1f284f KRW	140	check_iommu_entries(__iommu_table, __iommu_table_end);
116890d5	141
ee1f284f KRW	142	for (p = __iommu_table; p < __iommu_table_end; p++) {
	143	if (p && p->detect && p->detect() > 0) {
	144	p->flags \|= IOMMU_DETECTED;
	145	if (p->early_init)
	146	p->early_init();
	147	if (p->flags & IOMMU_FINISH_IF_DETECTED)
	148	break;
	149	}
	150	}
116890d5	151	}
9f6ac577 FT	152	void dma_generic_alloc_coherent(struct device dev, size_t size,
	153	dma_addr_t *dma_addr, gfp_t flag)
	154	{
	155	unsigned long dma_mask;
	156	struct page *page;
	157	dma_addr_t addr;
	158
	159	dma_mask = dma_alloc_coherent_mask(dev, flag);
	160
	161	flag \|= __GFP_ZERO;
	162	again:
	163	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	164	if (!page)
	165	return NULL;
	166
	167	addr = page_to_phys(page);
a4c2baa6	168	if (addr + size > dma_mask) {
9f6ac577 FT	169	__free_pages(page, get_order(size));
9f6ac577 FT	170
284901a9	171	if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
9f6ac577 FT	172	flag = (flag & ~GFP_DMA32) \| GFP_DMA;
	173	goto again;
	174	}
	175
	176	return NULL;
	177	}
	178
	179	*dma_addr = addr;
	180	return page_address(page);
	181	}
	182
fae9a0d8 GC	183	/*
	184	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	185	* documentation.
	186	*/
	187	static __init int iommu_setup(char *p)
	188	{
	189	iommu_merge = 1;
	190
	191	if (!p)
	192	return -EINVAL;
	193
	194	while (*p) {
	195	if (!strncmp(p, "off", 3))
	196	no_iommu = 1;
	197	/* gart_parse_options has more force support */
	198	if (!strncmp(p, "force", 5))
	199	force_iommu = 1;
	200	if (!strncmp(p, "noforce", 7)) {
	201	iommu_merge = 0;
	202	force_iommu = 0;
	203	}
	204
	205	if (!strncmp(p, "biomerge", 8)) {
fae9a0d8 GC	206	iommu_merge = 1;
	207	force_iommu = 1;
	208	}
	209	if (!strncmp(p, "panic", 5))
	210	panic_on_overflow = 1;
	211	if (!strncmp(p, "nopanic", 7))
	212	panic_on_overflow = 0;
	213	if (!strncmp(p, "merge", 5)) {
	214	iommu_merge = 1;
	215	force_iommu = 1;
	216	}
	217	if (!strncmp(p, "nomerge", 7))
	218	iommu_merge = 0;
	219	if (!strncmp(p, "forcesac", 8))
	220	iommu_sac_force = 1;
	221	if (!strncmp(p, "allowdac", 8))
	222	forbid_dac = 0;
	223	if (!strncmp(p, "nodac", 5))
2ae8bb75	224	forbid_dac = 1;
fae9a0d8 GC	225	if (!strncmp(p, "usedac", 6)) {
	226	forbid_dac = -1;
	227	return 1;
	228	}
	229	#ifdef CONFIG_SWIOTLB
	230	if (!strncmp(p, "soft", 4))
	231	swiotlb = 1;
3238c0c4	232	#endif
80286879	233	if (!strncmp(p, "pt", 2))
4ed0d3e6	234	iommu_pass_through = 1;
fae9a0d8	235
fae9a0d8	236	gart_parse_options(p);
fae9a0d8 GC	237
	238	#ifdef CONFIG_CALGARY_IOMMU
	239	if (!strncmp(p, "calgary", 7))
	240	use_calgary = 1;
	241	#endif /* CONFIG_CALGARY_IOMMU */
	242
	243	p += strcspn(p, ",");
	244	if (*p == ',')
	245	++p;
	246	}
	247	return 0;
	248	}
	249	early_param("iommu", iommu_setup);
	250
8e0c3797 GC	251	int dma_supported(struct device *dev, u64 mask)
8e0c3797 GC	252	{
160c1d8e	253	struct dma_map_ops *ops = get_dma_ops(dev);
8d8bb39b	254
8e0c3797 GC	255	#ifdef CONFIG_PCI
8e0c3797 GC	256	if (mask > 0xffffffff && forbid_dac > 0) {
fc3a8828	257	dev_info(dev, "PCI: Disallowing DAC for device\n");
8e0c3797 GC	258	return 0;
	259	}
	260	#endif
	261
8d8bb39b FT	262	if (ops->dma_supported)
8d8bb39b FT	263	return ops->dma_supported(dev, mask);
8e0c3797 GC	264
	265	/* Copied from i386. Doesn't make much sense, because it will
	266	only work for pci_alloc_coherent.
	267	The caller just has to use GFP_DMA in this case. */
2f4f27d4	268	if (mask < DMA_BIT_MASK(24))
8e0c3797 GC	269	return 0;
	270
	271	/* Tell the device to use SAC when IOMMU force is on. This
	272	allows the driver to use cheaper accesses in some cases.
	273
	274	Problem with this is that if we overflow the IOMMU area and
	275	return DAC as fallback address the device may not handle it
	276	correctly.
	277
	278	As a special case some controllers have a 39bit address
	279	mode that is as efficient as 32bit (aic79xx). Don't force
	280	SAC for these. Assume all masks <= 40 bits are of this
	281	type. Normally this doesn't make any difference, but gives
	282	more gentle handling of IOMMU overflow. */
50cf156a	283	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
fc3a8828	284	dev_info(dev, "Force SAC with mask %Lx\n", mask);
8e0c3797 GC	285	return 0;
	286	}
	287
	288	return 1;
	289	}
	290	EXPORT_SYMBOL(dma_supported);
	291
cb5867a5 GC	292	static int __init pci_iommu_init(void)
cb5867a5 GC	293	{
ee1f284f	294	struct iommu_table_entry *p;
2118d0c5 JR	295	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
2118d0c5 JR	296
86f31952 JR	297	#ifdef CONFIG_PCI
	298	dma_debug_add_bus(&pci_bus_type);
	299	#endif
d07c1be0 FT	300	x86_init.iommu.iommu_init();
d07c1be0 FT	301
ee1f284f KRW	302	for (p = __iommu_table; p < __iommu_table_end; p++) {
	303	if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
	304	p->late_init();
	305	}
75f1cdf1	306
cb5867a5 GC	307	return 0;
cb5867a5 GC	308	}
cb5867a5	309	/* Must execute after PCI subsystem */
9a821b23	310	rootfs_initcall(pci_iommu_init);
3b15e581 FY	311
	312	#ifdef CONFIG_PCI
	313	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	314
	315	static __devinit void via_no_dac(struct pci_dev *dev)
	316	{
	317	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
13bf7576	318	dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
3b15e581 FY	319	forbid_dac = 1;
	320	}
	321	}
	322	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	323	#endif