[deliverable/linux.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dmar.h>
#include <linux/bootmem.h>
#include <linux/pci.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/iommu.h>
#include <asm/calgary.h>
#include <asm/amd_iommu.h>

static int forbid_dac __read_mostly;

struct dma_mapping_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/* This tells the BIO block layer to assume merging. Default to off
   because we cannot guarantee merging later. */
int iommu_bio_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_bio_merge);

dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);

/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to older i386. */
struct device fallback_dev = {
	.bus_id = "fallback device",
	.coherent_dma_mask = DMA_32BIT_MASK,
	.dma_mask = &fallback_dev.coherent_dma_mask,
};

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

#ifdef CONFIG_X86_64
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
	if (max_pfn <= MAX_DMA32_PFN)
		return;

	/*
	 * check aperture_64.c allocate_aperture() for reason about
	 * using 512M as goal
	 */
	align = 64ULL<<20;
	size = round_up(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
				 512ULL<<20);
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{

	if (max_pfn <= MAX_DMA32_PFN)
		return;

	if (!dma32_bootmem_ptr)
		return;

	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}

void __init pci_iommu_alloc(void)
{
	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();
	/*
	 * The order of these functions is important for
	 * fall-back/fail-over reasons
	 */
	gart_iommu_hole_init();

	detect_calgary();

	detect_intel_iommu();

	amd_iommu_detect();

	pci_swiotlb_init();
}

unsigned long iommu_num_pages(unsigned long addr, unsigned long len)
{
	unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);

	return size >> PAGE_SHIFT;
}
EXPORT_SYMBOL(iommu_num_pages);
#endif

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_bio_merge = 4096;
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = -1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

int dma_supported(struct device *dev, u64 mask)
{
	struct dma_mapping_ops *ops = get_dma_ops(dev);

#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		dev_info(dev, "PCI: Disallowing DAC for device\n");
		return 0;
	}
#endif

	if (ops->dma_supported)
		return ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_24BIT_MASK)
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

/*
 * Allocate memory for a coherent mapping.
 */
	void *
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
		   gfp_t gfp)
{
	struct dma_mapping_ops *ops = get_dma_ops(dev);
	void *memory;

	if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
		return memory;

	if (!dev) {
		dev = &fallback_dev;
		gfp |= GFP_DMA;
	}

	if (ops->alloc_coherent)
		return ops->alloc_coherent(dev, size,
				dma_handle, gfp);
	return NULL;
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Unmap coherent memory.
 * The caller must ensure that the device has finished accessing the mapping.
 */
void dma_free_coherent(struct device *dev, size_t size,
		       void *vaddr, dma_addr_t bus)
{
	struct dma_mapping_ops *ops = get_dma_ops(dev);

	WARN_ON(irqs_disabled());       /* for portability */

	if (dma_release_from_coherent(dev, get_order(size), vaddr))
		return;

	if (ops->free_coherent)
		ops->free_coherent(dev, size, vaddr, bus);
}
EXPORT_SYMBOL(dma_free_coherent);

static int __init pci_iommu_init(void)
{
	calgary_iommu_init();

	intel_iommu_init();

	amd_iommu_init();

	gart_iommu_init();

	no_iommu_init();
	return 0;
}

void pci_iommu_shutdown(void)
{
	gart_iommu_shutdown();
}
/* Must execute after PCI subsystem */
fs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		printk(KERN_INFO "PCI: VIA PCI bridge detected."
				 "Disabling DAC.\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
cb5867a5	2	#include <linux/dmar.h>
116890d5	3	#include <linux/bootmem.h>
bca5c096	4	#include <linux/pci.h>
cb5867a5	5
116890d5 GC	6	#include <asm/proto.h>
116890d5 GC	7	#include <asm/dma.h>
46a7fa27	8	#include <asm/iommu.h>
cb5867a5	9	#include <asm/calgary.h>
a69ca340	10	#include <asm/amd_iommu.h>
459121c9	11
08e1a13e	12	static int forbid_dac __read_mostly;
bca5c096	13
8d8bb39b	14	struct dma_mapping_ops *dma_ops;
85c246ee GC	15	EXPORT_SYMBOL(dma_ops);
85c246ee GC	16
b4cdc430	17	static int iommu_sac_force __read_mostly;
8e0c3797	18
f9c258de GC	19	#ifdef CONFIG_IOMMU_DEBUG
	20	int panic_on_overflow __read_mostly = 1;
	21	int force_iommu __read_mostly = 1;
	22	#else
	23	int panic_on_overflow __read_mostly = 0;
	24	int force_iommu __read_mostly = 0;
	25	#endif
	26
fae9a0d8 GC	27	int iommu_merge __read_mostly = 0;
	28
	29	int no_iommu __read_mostly;
	30	/* Set this to 1 if there is a HW IOMMU in the system */
	31	int iommu_detected __read_mostly = 0;
	32
	33	/* This tells the BIO block layer to assume merging. Default to off
	34	because we cannot guarantee merging later. */
	35	int iommu_bio_merge __read_mostly = 0;
	36	EXPORT_SYMBOL(iommu_bio_merge);
	37
cac67877 GC	38	dma_addr_t bad_dma_address __read_mostly = 0;
cac67877 GC	39	EXPORT_SYMBOL(bad_dma_address);
fae9a0d8	40
098cb7f2 GC	41	/* Dummy device used for NULL arguments (normally ISA). Better would
	42	be probably a smaller DMA mask, but this is bug-to-bug compatible
	43	to older i386. */
	44	struct device fallback_dev = {
	45	.bus_id = "fallback device",
	46	.coherent_dma_mask = DMA_32BIT_MASK,
	47	.dma_mask = &fallback_dev.coherent_dma_mask,
	48	};
	49
459121c9 GC	50	int dma_set_mask(struct device *dev, u64 mask)
	51	{
	52	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	53	return -EIO;
	54
	55	*dev->dma_mask = mask;
	56
	57	return 0;
	58	}
	59	EXPORT_SYMBOL(dma_set_mask);
	60
116890d5 GC	61	#ifdef CONFIG_X86_64
	62	static __initdata void *dma32_bootmem_ptr;
	63	static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
	64
	65	static int __init parse_dma32_size_opt(char *p)
	66	{
	67	if (!p)
	68	return -EINVAL;
	69	dma32_bootmem_size = memparse(p, &p);
	70	return 0;
	71	}
	72	early_param("dma32_size", parse_dma32_size_opt);
	73
	74	void __init dma32_reserve_bootmem(void)
	75	{
	76	unsigned long size, align;
c987d12f	77	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	78	return;
116890d5 GC	79
7677b2ef YL	80	/*
	81	* check aperture_64.c allocate_aperture() for reason about
	82	* using 512M as goal
	83	*/
116890d5 GC	84	align = 64ULL<<20;
	85	size = round_up(dma32_bootmem_size, align);
	86	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
7677b2ef	87	512ULL<<20);
116890d5 GC	88	if (dma32_bootmem_ptr)
	89	dma32_bootmem_size = size;
	90	else
	91	dma32_bootmem_size = 0;
	92	}
	93	static void __init dma32_free_bootmem(void)
	94	{
116890d5	95
c987d12f	96	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	97	return;
	98
	99	if (!dma32_bootmem_ptr)
	100	return;
	101
330fce23	102	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
116890d5 GC	103
	104	dma32_bootmem_ptr = NULL;
	105	dma32_bootmem_size = 0;
	106	}
	107
	108	void __init pci_iommu_alloc(void)
	109	{
	110	/* free the range so iommu could get some range less than 4G */
	111	dma32_free_bootmem();
	112	/*
	113	* The order of these functions is important for
	114	* fall-back/fail-over reasons
	115	*/
116890d5	116	gart_iommu_hole_init();
116890d5	117
116890d5	118	detect_calgary();
116890d5 GC	119
	120	detect_intel_iommu();
	121
a69ca340 JR	122	amd_iommu_detect();
a69ca340 JR	123
116890d5	124	pci_swiotlb_init();
116890d5	125	}
8978b742 FT	126
	127	unsigned long iommu_num_pages(unsigned long addr, unsigned long len)
	128	{
	129	unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);
	130
	131	return size >> PAGE_SHIFT;
	132	}
	133	EXPORT_SYMBOL(iommu_num_pages);
116890d5 GC	134	#endif
116890d5 GC	135
fae9a0d8 GC	136	/*
	137	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	138	* documentation.
	139	*/
	140	static __init int iommu_setup(char *p)
	141	{
	142	iommu_merge = 1;
	143
	144	if (!p)
	145	return -EINVAL;
	146
	147	while (*p) {
	148	if (!strncmp(p, "off", 3))
	149	no_iommu = 1;
	150	/* gart_parse_options has more force support */
	151	if (!strncmp(p, "force", 5))
	152	force_iommu = 1;
	153	if (!strncmp(p, "noforce", 7)) {
	154	iommu_merge = 0;
	155	force_iommu = 0;
	156	}
	157
	158	if (!strncmp(p, "biomerge", 8)) {
	159	iommu_bio_merge = 4096;
	160	iommu_merge = 1;
	161	force_iommu = 1;
	162	}
	163	if (!strncmp(p, "panic", 5))
	164	panic_on_overflow = 1;
	165	if (!strncmp(p, "nopanic", 7))
	166	panic_on_overflow = 0;
	167	if (!strncmp(p, "merge", 5)) {
	168	iommu_merge = 1;
	169	force_iommu = 1;
	170	}
	171	if (!strncmp(p, "nomerge", 7))
	172	iommu_merge = 0;
	173	if (!strncmp(p, "forcesac", 8))
	174	iommu_sac_force = 1;
	175	if (!strncmp(p, "allowdac", 8))
	176	forbid_dac = 0;
	177	if (!strncmp(p, "nodac", 5))
	178	forbid_dac = -1;
	179	if (!strncmp(p, "usedac", 6)) {
	180	forbid_dac = -1;
	181	return 1;
	182	}
	183	#ifdef CONFIG_SWIOTLB
	184	if (!strncmp(p, "soft", 4))
	185	swiotlb = 1;
	186	#endif
	187
fae9a0d8	188	gart_parse_options(p);
fae9a0d8 GC	189
	190	#ifdef CONFIG_CALGARY_IOMMU
	191	if (!strncmp(p, "calgary", 7))
	192	use_calgary = 1;
	193	#endif /* CONFIG_CALGARY_IOMMU */
	194
	195	p += strcspn(p, ",");
	196	if (*p == ',')
	197	++p;
	198	}
	199	return 0;
	200	}
	201	early_param("iommu", iommu_setup);
	202
8e0c3797 GC	203	int dma_supported(struct device *dev, u64 mask)
8e0c3797 GC	204	{
8d8bb39b FT	205	struct dma_mapping_ops *ops = get_dma_ops(dev);
8d8bb39b FT	206
8e0c3797 GC	207	#ifdef CONFIG_PCI
8e0c3797 GC	208	if (mask > 0xffffffff && forbid_dac > 0) {
fc3a8828	209	dev_info(dev, "PCI: Disallowing DAC for device\n");
8e0c3797 GC	210	return 0;
	211	}
	212	#endif
	213
8d8bb39b FT	214	if (ops->dma_supported)
8d8bb39b FT	215	return ops->dma_supported(dev, mask);
8e0c3797 GC	216
	217	/* Copied from i386. Doesn't make much sense, because it will
	218	only work for pci_alloc_coherent.
	219	The caller just has to use GFP_DMA in this case. */
	220	if (mask < DMA_24BIT_MASK)
	221	return 0;
	222
	223	/* Tell the device to use SAC when IOMMU force is on. This
	224	allows the driver to use cheaper accesses in some cases.
	225
	226	Problem with this is that if we overflow the IOMMU area and
	227	return DAC as fallback address the device may not handle it
	228	correctly.
	229
	230	As a special case some controllers have a 39bit address
	231	mode that is as efficient as 32bit (aic79xx). Don't force
	232	SAC for these. Assume all masks <= 40 bits are of this
	233	type. Normally this doesn't make any difference, but gives
	234	more gentle handling of IOMMU overflow. */
	235	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
fc3a8828	236	dev_info(dev, "Force SAC with mask %Lx\n", mask);
8e0c3797 GC	237	return 0;
	238	}
	239
	240	return 1;
	241	}
	242	EXPORT_SYMBOL(dma_supported);
	243
098cb7f2 GC	244	/*
	245	* Allocate memory for a coherent mapping.
	246	*/
c647c3bb	247	void *
098cb7f2 GC	248	dma_alloc_coherent(struct device dev, size_t size, dma_addr_t dma_handle,
	249	gfp_t gfp)
	250	{
8d8bb39b	251	struct dma_mapping_ops *ops = get_dma_ops(dev);
c647c3bb	252	void *memory;
098cb7f2	253
323ec001	254	if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
098cb7f2 GC	255	return memory;
098cb7f2 GC	256
4a367f3a	257	if (!dev) {
098cb7f2	258	dev = &fallback_dev;
4a367f3a TI	259	gfp \|= GFP_DMA;
4a367f3a TI	260	}
098cb7f2	261
c647c3bb JR	262	if (ops->alloc_coherent)
	263	return ops->alloc_coherent(dev, size,
	264	dma_handle, gfp);
098cb7f2 GC	265	return NULL;
	266	}
	267	EXPORT_SYMBOL(dma_alloc_coherent);
	268
	269	/*
	270	* Unmap coherent memory.
	271	* The caller must ensure that the device has finished accessing the mapping.
	272	*/
	273	void dma_free_coherent(struct device *dev, size_t size,
c647c3bb	274	void *vaddr, dma_addr_t bus)
098cb7f2	275	{
8d8bb39b FT	276	struct dma_mapping_ops *ops = get_dma_ops(dev);
8d8bb39b FT	277
c647c3bb JR	278	WARN_ON(irqs_disabled()); /* for portability */
	279
	280	if (dma_release_from_coherent(dev, get_order(size), vaddr))
098cb7f2	281	return;
c647c3bb JR	282
	283	if (ops->free_coherent)
	284	ops->free_coherent(dev, size, vaddr, bus);
098cb7f2 GC	285	}
098cb7f2 GC	286	EXPORT_SYMBOL(dma_free_coherent);
8e0c3797	287
cb5867a5 GC	288	static int __init pci_iommu_init(void)
cb5867a5 GC	289	{
cb5867a5	290	calgary_iommu_init();
cb5867a5 GC	291
	292	intel_iommu_init();
	293
a69ca340 JR	294	amd_iommu_init();
a69ca340 JR	295
cb5867a5	296	gart_iommu_init();
459121c9	297
cb5867a5 GC	298	no_iommu_init();
	299	return 0;
	300	}
	301
	302	void pci_iommu_shutdown(void)
	303	{
	304	gart_iommu_shutdown();
	305	}
	306	/* Must execute after PCI subsystem */
	307	fs_initcall(pci_iommu_init);
bca5c096 GC	308
	309	#ifdef CONFIG_PCI
	310	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	311
	312	static __devinit void via_no_dac(struct pci_dev *dev)
	313	{
	314	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
	315	printk(KERN_INFO "PCI: VIA PCI bridge detected."
	316	"Disabling DAC.\n");
	317	forbid_dac = 1;
	318	}
	319	}
	320	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	321	#endif