[deliverable/linux.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dmar.h>
#include <linux/bootmem.h>
#include <linux/pci.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/gart.h>
#include <asm/calgary.h>

int forbid_dac __read_mostly;
EXPORT_SYMBOL(forbid_dac);

const struct dma_mapping_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/* This tells the BIO block layer to assume merging. Default to off
   because we cannot guarantee merging later. */
int iommu_bio_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_bio_merge);

dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);

/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to older i386. */
struct device fallback_dev = {
	.bus_id = "fallback device",
	.coherent_dma_mask = DMA_32BIT_MASK,
	.dma_mask = &fallback_dev.coherent_dma_mask,
};

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

#ifdef CONFIG_X86_64
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
	if (end_pfn <= MAX_DMA32_PFN)
		return;

	/*
	 * check aperture_64.c allocate_aperture() for reason about
	 * using 512M as goal
	 */
	align = 64ULL<<20;
	size = round_up(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
				 512ULL<<20);
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{
	int node;

	if (end_pfn <= MAX_DMA32_PFN)
		return;

	if (!dma32_bootmem_ptr)
		return;

	for_each_online_node(node)
		free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr),
				  dma32_bootmem_size);

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}

void __init pci_iommu_alloc(void)
{
	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();
	/*
	 * The order of these functions is important for
	 * fall-back/fail-over reasons
	 */
#ifdef CONFIG_GART_IOMMU
	gart_iommu_hole_init();
#endif

#ifdef CONFIG_CALGARY_IOMMU
	detect_calgary();
#endif

	detect_intel_iommu();

#ifdef CONFIG_SWIOTLB
	pci_swiotlb_init();
#endif
}
#endif

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_bio_merge = 4096;
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = -1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif

#ifdef CONFIG_GART_IOMMU
		gart_parse_options(p);
#endif

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

#ifdef CONFIG_X86_32
int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
				dma_addr_t device_addr, size_t size, int flags)
{
	void __iomem *mem_base = NULL;
	int pages = size >> PAGE_SHIFT;
	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);

	if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
		goto out;
	if (!size)
		goto out;
	if (dev->dma_mem)
		goto out;

	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */

	mem_base = ioremap(bus_addr, size);
	if (!mem_base)
		goto out;

	dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	if (!dev->dma_mem)
		goto out;
	dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	if (!dev->dma_mem->bitmap)
		goto free1_out;

	dev->dma_mem->virt_base = mem_base;
	dev->dma_mem->device_base = device_addr;
	dev->dma_mem->size = pages;
	dev->dma_mem->flags = flags;

	if (flags & DMA_MEMORY_MAP)
		return DMA_MEMORY_MAP;

	return DMA_MEMORY_IO;

 free1_out:
	kfree(dev->dma_mem);
 out:
	if (mem_base)
		iounmap(mem_base);
	return 0;
}
EXPORT_SYMBOL(dma_declare_coherent_memory);

void dma_release_declared_memory(struct device *dev)
{
	struct dma_coherent_mem *mem = dev->dma_mem;

	if (!mem)
		return;
	dev->dma_mem = NULL;
	iounmap(mem->virt_base);
	kfree(mem->bitmap);
	kfree(mem);
}
EXPORT_SYMBOL(dma_release_declared_memory);

void *dma_mark_declared_memory_occupied(struct device *dev,
					dma_addr_t device_addr, size_t size)
{
	struct dma_coherent_mem *mem = dev->dma_mem;
	int pos, err;
	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);

	pages >>= PAGE_SHIFT;

	if (!mem)
		return ERR_PTR(-EINVAL);

	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
	if (err != 0)
		return ERR_PTR(err);
	return mem->virt_base + (pos << PAGE_SHIFT);
}
EXPORT_SYMBOL(dma_mark_declared_memory_occupied);

static int dma_alloc_from_coherent_mem(struct device *dev, ssize_t size,
				       dma_addr_t *dma_handle, void **ret)
{
	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	int order = get_order(size);

	if (mem) {
		int page = bitmap_find_free_region(mem->bitmap, mem->size,
						     order);
		if (page >= 0) {
			*dma_handle = mem->device_base + (page << PAGE_SHIFT);
			*ret = mem->virt_base + (page << PAGE_SHIFT);
			memset(*ret, 0, size);
		}
		if (mem->flags & DMA_MEMORY_EXCLUSIVE)
			*ret = NULL;
	}
	return (mem != NULL);
}

static int dma_release_coherent(struct device *dev, int order, void *vaddr)
{
	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;

	if (mem && vaddr >= mem->virt_base && vaddr <
		   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;

		bitmap_release_region(mem->bitmap, page, order);
		return 1;
	}
	return 0;
}
#else
#define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
#define dma_release_coherent(dev, order, vaddr) (0)
#endif /* CONFIG_X86_32 */

int dma_supported(struct device *dev, u64 mask)
{
#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		printk(KERN_INFO "PCI: Disallowing DAC for device %s\n",
				 dev->bus_id);
		return 0;
	}
#endif

	if (dma_ops->dma_supported)
		return dma_ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_24BIT_MASK)
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
		printk(KERN_INFO "%s: Force SAC with mask %Lx\n",
				 dev->bus_id, mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

/* Allocate DMA memory on node near device */
noinline struct page *
dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
{
	int node;

	node = dev_to_node(dev);

	return alloc_pages_node(node, gfp, order);
}

/*
 * Allocate memory for a coherent mapping.
 */
void *
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
		   gfp_t gfp)
{
	void *memory = NULL;
	struct page *page;
	unsigned long dma_mask = 0;
	dma_addr_t bus;

	/* ignore region specifiers */
	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);

	if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
		return memory;

	if (!dev)
		dev = &fallback_dev;
	dma_mask = dev->coherent_dma_mask;
	if (dma_mask == 0)
		dma_mask = DMA_32BIT_MASK;

	/* Device not DMA able */
	if (dev->dma_mask == NULL)
		return NULL;

	/* Don't invoke OOM killer */
	gfp |= __GFP_NORETRY;

#ifdef CONFIG_X86_64
	/* Why <=? Even when the mask is smaller than 4GB it is often
	   larger than 16MB and in this case we have a chance of
	   finding fitting memory in the next higher zone first. If
	   not retry with true GFP_DMA. -AK */
	if (dma_mask <= DMA_32BIT_MASK)
		gfp |= GFP_DMA32;
#endif

 again:
	page = dma_alloc_pages(dev, gfp, get_order(size));
	if (page == NULL)
		return NULL;

	{
		int high, mmu;
		bus = page_to_phys(page);
		memory = page_address(page);
		high = (bus + size) >= dma_mask;
		mmu = high;
		if (force_iommu && !(gfp & GFP_DMA))
			mmu = 1;
		else if (high) {
			free_pages((unsigned long)memory,
				   get_order(size));

			/* Don't use the 16MB ZONE_DMA unless absolutely
			   needed. It's better to use remapping first. */
			if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
				gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
				goto again;
			}

			/* Let low level make its own zone decisions */
			gfp &= ~(GFP_DMA32|GFP_DMA);

			if (dma_ops->alloc_coherent)
				return dma_ops->alloc_coherent(dev, size,
							   dma_handle, gfp);
			return NULL;
		}

		memset(memory, 0, size);
		if (!mmu) {
			*dma_handle = bus;
			return memory;
		}
	}

	if (dma_ops->alloc_coherent) {
		free_pages((unsigned long)memory, get_order(size));
		gfp &= ~(GFP_DMA|GFP_DMA32);
		return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
	}

	if (dma_ops->map_simple) {
		*dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
					      size,
					      PCI_DMA_BIDIRECTIONAL);
		if (*dma_handle != bad_dma_address)
			return memory;
	}

	if (panic_on_overflow)
		panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
		      (unsigned long)size);
	free_pages((unsigned long)memory, get_order(size));
	return NULL;
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Unmap coherent memory.
 * The caller must ensure that the device has finished accessing the mapping.
 */
void dma_free_coherent(struct device *dev, size_t size,
			 void *vaddr, dma_addr_t bus)
{
	int order = get_order(size);
	WARN_ON(irqs_disabled());	/* for portability */
	if (dma_release_coherent(dev, order, vaddr))
		return;
	if (dma_ops->unmap_single)
		dma_ops->unmap_single(dev, bus, size, 0);
	free_pages((unsigned long)vaddr, order);
}
EXPORT_SYMBOL(dma_free_coherent);

static int __init pci_iommu_init(void)
{
#ifdef CONFIG_CALGARY_IOMMU
	calgary_iommu_init();
#endif

	intel_iommu_init();

#ifdef CONFIG_GART_IOMMU
	gart_iommu_init();
#endif

	no_iommu_init();
	return 0;
}

void pci_iommu_shutdown(void)
{
	gart_iommu_shutdown();
}
/* Must execute after PCI subsystem */
fs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		printk(KERN_INFO "PCI: VIA PCI bridge detected."
				 "Disabling DAC.\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
cb5867a5	2	#include <linux/dmar.h>
116890d5	3	#include <linux/bootmem.h>
bca5c096	4	#include <linux/pci.h>
cb5867a5	5
116890d5 GC	6	#include <asm/proto.h>
116890d5 GC	7	#include <asm/dma.h>
cb5867a5 GC	8	#include <asm/gart.h>
cb5867a5 GC	9	#include <asm/calgary.h>
459121c9	10
bca5c096 GC	11	int forbid_dac __read_mostly;
	12	EXPORT_SYMBOL(forbid_dac);
	13
85c246ee GC	14	const struct dma_mapping_ops *dma_ops;
	15	EXPORT_SYMBOL(dma_ops);
	16
b4cdc430	17	static int iommu_sac_force __read_mostly;
8e0c3797	18
f9c258de GC	19	#ifdef CONFIG_IOMMU_DEBUG
	20	int panic_on_overflow __read_mostly = 1;
	21	int force_iommu __read_mostly = 1;
	22	#else
	23	int panic_on_overflow __read_mostly = 0;
	24	int force_iommu __read_mostly = 0;
	25	#endif
	26
fae9a0d8 GC	27	int iommu_merge __read_mostly = 0;
	28
	29	int no_iommu __read_mostly;
	30	/* Set this to 1 if there is a HW IOMMU in the system */
	31	int iommu_detected __read_mostly = 0;
	32
	33	/* This tells the BIO block layer to assume merging. Default to off
	34	because we cannot guarantee merging later. */
	35	int iommu_bio_merge __read_mostly = 0;
	36	EXPORT_SYMBOL(iommu_bio_merge);
	37
cac67877 GC	38	dma_addr_t bad_dma_address __read_mostly = 0;
cac67877 GC	39	EXPORT_SYMBOL(bad_dma_address);
fae9a0d8	40
098cb7f2 GC	41	/* Dummy device used for NULL arguments (normally ISA). Better would
	42	be probably a smaller DMA mask, but this is bug-to-bug compatible
	43	to older i386. */
	44	struct device fallback_dev = {
	45	.bus_id = "fallback device",
	46	.coherent_dma_mask = DMA_32BIT_MASK,
	47	.dma_mask = &fallback_dev.coherent_dma_mask,
	48	};
	49
459121c9 GC	50	int dma_set_mask(struct device *dev, u64 mask)
	51	{
	52	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	53	return -EIO;
	54
	55	*dev->dma_mask = mask;
	56
	57	return 0;
	58	}
	59	EXPORT_SYMBOL(dma_set_mask);
	60
116890d5 GC	61	#ifdef CONFIG_X86_64
	62	static __initdata void *dma32_bootmem_ptr;
	63	static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
	64
	65	static int __init parse_dma32_size_opt(char *p)
	66	{
	67	if (!p)
	68	return -EINVAL;
	69	dma32_bootmem_size = memparse(p, &p);
	70	return 0;
	71	}
	72	early_param("dma32_size", parse_dma32_size_opt);
	73
	74	void __init dma32_reserve_bootmem(void)
	75	{
	76	unsigned long size, align;
	77	if (end_pfn <= MAX_DMA32_PFN)
	78	return;
	79
7677b2ef YL	80	/*
	81	* check aperture_64.c allocate_aperture() for reason about
	82	* using 512M as goal
	83	*/
116890d5 GC	84	align = 64ULL<<20;
	85	size = round_up(dma32_bootmem_size, align);
	86	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
7677b2ef	87	512ULL<<20);
116890d5 GC	88	if (dma32_bootmem_ptr)
	89	dma32_bootmem_size = size;
	90	else
	91	dma32_bootmem_size = 0;
	92	}
	93	static void __init dma32_free_bootmem(void)
	94	{
	95	int node;
	96
	97	if (end_pfn <= MAX_DMA32_PFN)
	98	return;
	99
	100	if (!dma32_bootmem_ptr)
	101	return;
	102
	103	for_each_online_node(node)
	104	free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr),
	105	dma32_bootmem_size);
	106
	107	dma32_bootmem_ptr = NULL;
	108	dma32_bootmem_size = 0;
	109	}
	110
	111	void __init pci_iommu_alloc(void)
	112	{
	113	/* free the range so iommu could get some range less than 4G */
	114	dma32_free_bootmem();
	115	/*
	116	* The order of these functions is important for
	117	* fall-back/fail-over reasons
	118	*/
	119	#ifdef CONFIG_GART_IOMMU
	120	gart_iommu_hole_init();
	121	#endif
	122
	123	#ifdef CONFIG_CALGARY_IOMMU
	124	detect_calgary();
	125	#endif
	126
	127	detect_intel_iommu();
	128
	129	#ifdef CONFIG_SWIOTLB
	130	pci_swiotlb_init();
	131	#endif
	132	}
	133	#endif
	134
fae9a0d8 GC	135	/*
	136	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	137	* documentation.
	138	*/
	139	static __init int iommu_setup(char *p)
	140	{
	141	iommu_merge = 1;
	142
	143	if (!p)
	144	return -EINVAL;
	145
	146	while (*p) {
	147	if (!strncmp(p, "off", 3))
	148	no_iommu = 1;
	149	/* gart_parse_options has more force support */
	150	if (!strncmp(p, "force", 5))
	151	force_iommu = 1;
	152	if (!strncmp(p, "noforce", 7)) {
	153	iommu_merge = 0;
	154	force_iommu = 0;
	155	}
	156
	157	if (!strncmp(p, "biomerge", 8)) {
	158	iommu_bio_merge = 4096;
	159	iommu_merge = 1;
	160	force_iommu = 1;
	161	}
	162	if (!strncmp(p, "panic", 5))
	163	panic_on_overflow = 1;
	164	if (!strncmp(p, "nopanic", 7))
	165	panic_on_overflow = 0;
	166	if (!strncmp(p, "merge", 5)) {
	167	iommu_merge = 1;
	168	force_iommu = 1;
	169	}
	170	if (!strncmp(p, "nomerge", 7))
	171	iommu_merge = 0;
	172	if (!strncmp(p, "forcesac", 8))
	173	iommu_sac_force = 1;
	174	if (!strncmp(p, "allowdac", 8))
	175	forbid_dac = 0;
	176	if (!strncmp(p, "nodac", 5))
	177	forbid_dac = -1;
	178	if (!strncmp(p, "usedac", 6)) {
	179	forbid_dac = -1;
	180	return 1;
	181	}
	182	#ifdef CONFIG_SWIOTLB
	183	if (!strncmp(p, "soft", 4))
	184	swiotlb = 1;
	185	#endif
	186
	187	#ifdef CONFIG_GART_IOMMU
	188	gart_parse_options(p);
	189	#endif
	190
	191	#ifdef CONFIG_CALGARY_IOMMU
	192	if (!strncmp(p, "calgary", 7))
	193	use_calgary = 1;
	194	#endif /* CONFIG_CALGARY_IOMMU */
	195
	196	p += strcspn(p, ",");
	197	if (*p == ',')
	198	++p;
199	}
200	return 0;
201	}
202	early_param("iommu", iommu_setup);
203
8e8edc64 GC	204	#ifdef CONFIG_X86_32
	205	int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
	206	dma_addr_t device_addr, size_t size, int flags)
	207	{
	208	void __iomem *mem_base = NULL;
	209	int pages = size >> PAGE_SHIFT;
	210	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
	211
	212	if ((flags & (DMA_MEMORY_MAP \| DMA_MEMORY_IO)) == 0)
	213	goto out;
	214	if (!size)
	215	goto out;
	216	if (dev->dma_mem)
	217	goto out;
	218
	219	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
	220
	221	mem_base = ioremap(bus_addr, size);
	222	if (!mem_base)
	223	goto out;
	224
	225	dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	226	if (!dev->dma_mem)
	227	goto out;
	228	dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	229	if (!dev->dma_mem->bitmap)
	230	goto free1_out;
	231
	232	dev->dma_mem->virt_base = mem_base;
	233	dev->dma_mem->device_base = device_addr;
	234	dev->dma_mem->size = pages;
	235	dev->dma_mem->flags = flags;
	236
	237	if (flags & DMA_MEMORY_MAP)
	238	return DMA_MEMORY_MAP;
	239
	240	return DMA_MEMORY_IO;
	241
	242	free1_out:
	243	kfree(dev->dma_mem);
	244	out:
	245	if (mem_base)
	246	iounmap(mem_base);
	247	return 0;
	248	}
	249	EXPORT_SYMBOL(dma_declare_coherent_memory);
	250
	251	void dma_release_declared_memory(struct device *dev)
	252	{
	253	struct dma_coherent_mem *mem = dev->dma_mem;
	254
	255	if (!mem)
	256	return;
	257	dev->dma_mem = NULL;
	258	iounmap(mem->virt_base);
	259	kfree(mem->bitmap);
	260	kfree(mem);
	261	}
	262	EXPORT_SYMBOL(dma_release_declared_memory);
	263
	264	void dma_mark_declared_memory_occupied(struct device dev,
	265	dma_addr_t device_addr, size_t size)
	266	{
	267	struct dma_coherent_mem *mem = dev->dma_mem;
268	int pos, err;
269	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);
270
271	pages >>= PAGE_SHIFT;
272
273	if (!mem)
274	return ERR_PTR(-EINVAL);
275
276	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
277	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
278	if (err != 0)
279	return ERR_PTR(err);
280	return mem->virt_base + (pos << PAGE_SHIFT);
281	}
282	EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
098cb7f2 GC	283
	284	static int dma_alloc_from_coherent_mem(struct device *dev, ssize_t size,
	285	dma_addr_t dma_handle, void *ret)
	286	{
	287	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	288	int order = get_order(size);
	289
	290	if (mem) {
	291	int page = bitmap_find_free_region(mem->bitmap, mem->size,
	292	order);
	293	if (page >= 0) {
	294	*dma_handle = mem->device_base + (page << PAGE_SHIFT);
	295	*ret = mem->virt_base + (page << PAGE_SHIFT);
	296	memset(*ret, 0, size);
	297	}
	298	if (mem->flags & DMA_MEMORY_EXCLUSIVE)
	299	*ret = NULL;
	300	}
	301	return (mem != NULL);
	302	}
	303
	304	static int dma_release_coherent(struct device dev, int order, void vaddr)
	305	{
	306	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	307
	308	if (mem && vaddr >= mem->virt_base && vaddr <
	309	(mem->virt_base + (mem->size << PAGE_SHIFT))) {
	310	int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
	311
	312	bitmap_release_region(mem->bitmap, page, order);
	313	return 1;
	314	}
	315	return 0;
	316	}
	317	#else
	318	#define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
	319	#define dma_release_coherent(dev, order, vaddr) (0)
8e8edc64 GC	320	#endif /* CONFIG_X86_32 */
8e8edc64 GC	321
8e0c3797 GC	322	int dma_supported(struct device *dev, u64 mask)
	323	{
	324	#ifdef CONFIG_PCI
	325	if (mask > 0xffffffff && forbid_dac > 0) {
	326	printk(KERN_INFO "PCI: Disallowing DAC for device %s\n",
	327	dev->bus_id);
	328	return 0;
	329	}
	330	#endif
	331
	332	if (dma_ops->dma_supported)
	333	return dma_ops->dma_supported(dev, mask);
	334
	335	/* Copied from i386. Doesn't make much sense, because it will
	336	only work for pci_alloc_coherent.
	337	The caller just has to use GFP_DMA in this case. */
	338	if (mask < DMA_24BIT_MASK)
	339	return 0;
	340
	341	/* Tell the device to use SAC when IOMMU force is on. This
	342	allows the driver to use cheaper accesses in some cases.
	343
	344	Problem with this is that if we overflow the IOMMU area and
	345	return DAC as fallback address the device may not handle it
	346	correctly.
	347
	348	As a special case some controllers have a 39bit address
	349	mode that is as efficient as 32bit (aic79xx). Don't force
	350	SAC for these. Assume all masks <= 40 bits are of this
	351	type. Normally this doesn't make any difference, but gives
	352	more gentle handling of IOMMU overflow. */
	353	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
	354	printk(KERN_INFO "%s: Force SAC with mask %Lx\n",
	355	dev->bus_id, mask);
	356	return 0;
	357	}
	358
	359	return 1;
	360	}
	361	EXPORT_SYMBOL(dma_supported);
	362
098cb7f2 GC	363	/* Allocate DMA memory on node near device */
	364	noinline struct page *
	365	dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
	366	{
	367	int node;
	368
	369	node = dev_to_node(dev);
	370
	371	return alloc_pages_node(node, gfp, order);
	372	}
	373
	374	/*
	375	* Allocate memory for a coherent mapping.
	376	*/
	377	void *
	378	dma_alloc_coherent(struct device dev, size_t size, dma_addr_t dma_handle,
	379	gfp_t gfp)
	380	{
	381	void *memory = NULL;
	382	struct page *page;
	383	unsigned long dma_mask = 0;
	384	dma_addr_t bus;
	385
	386	/* ignore region specifiers */
	387	gfp &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);
	388
	389	if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
	390	return memory;
	391
	392	if (!dev)
	393	dev = &fallback_dev;
	394	dma_mask = dev->coherent_dma_mask;
	395	if (dma_mask == 0)
	396	dma_mask = DMA_32BIT_MASK;
	397
	398	/* Device not DMA able */
	399	if (dev->dma_mask == NULL)
	400	return NULL;
	401
	402	/* Don't invoke OOM killer */
	403	gfp \|= __GFP_NORETRY;
	404
	405	#ifdef CONFIG_X86_64
	406	/* Why <=? Even when the mask is smaller than 4GB it is often
	407	larger than 16MB and in this case we have a chance of
	408	finding fitting memory in the next higher zone first. If
	409	not retry with true GFP_DMA. -AK */
	410	if (dma_mask <= DMA_32BIT_MASK)
	411	gfp \|= GFP_DMA32;
	412	#endif
	413
	414	again:
	415	page = dma_alloc_pages(dev, gfp, get_order(size));
	416	if (page == NULL)
	417	return NULL;
	418
	419	{
	420	int high, mmu;
	421	bus = page_to_phys(page);
	422	memory = page_address(page);
	423	high = (bus + size) >= dma_mask;
	424	mmu = high;
	425	if (force_iommu && !(gfp & GFP_DMA))
	426	mmu = 1;
427	else if (high) {
428	free_pages((unsigned long)memory,
429	get_order(size));
430
431	/* Don't use the 16MB ZONE_DMA unless absolutely
432	needed. It's better to use remapping first. */
433	if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
434	gfp = (gfp & ~GFP_DMA32) \| GFP_DMA;
435	goto again;
436	}
437
438	/* Let low level make its own zone decisions */
439	gfp &= ~(GFP_DMA32\|GFP_DMA);
440
441	if (dma_ops->alloc_coherent)
442	return dma_ops->alloc_coherent(dev, size,
443	dma_handle, gfp);
444	return NULL;
445	}
446
447	memset(memory, 0, size);
448	if (!mmu) {
449	*dma_handle = bus;
450	return memory;
451	}
452	}
453
454	if (dma_ops->alloc_coherent) {
455	free_pages((unsigned long)memory, get_order(size));
456	gfp &= ~(GFP_DMA\|GFP_DMA32);
457	return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
458	}
459
460	if (dma_ops->map_simple) {
461	*dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
462	size,
463	PCI_DMA_BIDIRECTIONAL);
464	if (*dma_handle != bad_dma_address)
465	return memory;
466	}
467
468	if (panic_on_overflow)
469	panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
470	(unsigned long)size);
471	free_pages((unsigned long)memory, get_order(size));
472	return NULL;
473	}
474	EXPORT_SYMBOL(dma_alloc_coherent);
475
476	/*
477	* Unmap coherent memory.
478	* The caller must ensure that the device has finished accessing the mapping.
479	*/
480	void dma_free_coherent(struct device *dev, size_t size,
481	void *vaddr, dma_addr_t bus)
482	{
483	int order = get_order(size);
484	WARN_ON(irqs_disabled()); /* for portability */
485	if (dma_release_coherent(dev, order, vaddr))
486	return;
487	if (dma_ops->unmap_single)
488	dma_ops->unmap_single(dev, bus, size, 0);
489	free_pages((unsigned long)vaddr, order);
490	}
491	EXPORT_SYMBOL(dma_free_coherent);
8e0c3797	492
cb5867a5 GC	493	static int __init pci_iommu_init(void)
	494	{
	495	#ifdef CONFIG_CALGARY_IOMMU
	496	calgary_iommu_init();
	497	#endif
	498
	499	intel_iommu_init();
	500
	501	#ifdef CONFIG_GART_IOMMU
	502	gart_iommu_init();
	503	#endif
459121c9	504
cb5867a5 GC	505	no_iommu_init();
	506	return 0;
	507	}
	508
	509	void pci_iommu_shutdown(void)
	510	{
	511	gart_iommu_shutdown();
	512	}
	513	/* Must execute after PCI subsystem */
	514	fs_initcall(pci_iommu_init);
bca5c096 GC	515
	516	#ifdef CONFIG_PCI
	517	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	518
	519	static __devinit void via_no_dac(struct pci_dev *dev)
	520	{
	521	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
	522	printk(KERN_INFO "PCI: VIA PCI bridge detected."
	523	"Disabling DAC.\n");
	524	forbid_dac = 1;
	525	}
	526	}
	527	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	528	#endif