arch/x86/kernel/pci-dma_64.c

   1 /*
   2  * Dynamic DMA mapping support.
   3  */
   4
   5 #include <linux/types.h>
   6 #include <linux/mm.h>
   7 #include <linux/string.h>
   8 #include <linux/pci.h>
   9 #include <linux/module.h>
  10 #include <linux/dmar.h>
  11 #include <linux/bootmem.h>
  12 #include <asm/proto.h>
  13 #include <asm/io.h>
  14 #include <asm/gart.h>
  15 #include <asm/calgary.h>
  16
  17
  18 /* Dummy device used for NULL arguments (normally ISA). Better would
  19    be probably a smaller DMA mask, but this is bug-to-bug compatible
  20    to i386. */
  21 struct device fallback_dev = {
  22         .bus_id = "fallback device",
  23         .coherent_dma_mask = DMA_32BIT_MASK,
  24         .dma_mask = &fallback_dev.coherent_dma_mask,
  25 };
  26
  27 /* Allocate DMA memory on node near device */
  28 noinline static void *
  29 dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
  30 {
  31         int node;
  32
  33         node = dev_to_node(dev);
  34
  35         return alloc_pages_node(node, gfp, order);
  36 }
  37
  38 #define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
  39 #define dma_release_coherent(dev, order, vaddr) (0)
  40 /*
  41  * Allocate memory for a coherent mapping.
  42  */
  43 void *
  44 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
  45                    gfp_t gfp)
  46 {
  47         void *memory;
  48         struct page *page;
  49         unsigned long dma_mask = 0;
  50         u64 bus;
  51
  52
  53         if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
  54                 return memory;
  55
  56         if (!dev)
  57                 dev = &fallback_dev;
  58         dma_mask = dev->coherent_dma_mask;
  59         if (dma_mask == 0)
  60                 dma_mask = DMA_32BIT_MASK;
  61
  62         /* Device not DMA able */
  63         if (dev->dma_mask == NULL)
  64                 return NULL;
  65
  66         /* Don't invoke OOM killer */
  67         gfp |= __GFP_NORETRY;
  68
  69         /* Kludge to make it bug-to-bug compatible with i386. i386
  70            uses the normal dma_mask for alloc_coherent. */
  71         dma_mask &= *dev->dma_mask;
  72
  73         /* Why <=? Even when the mask is smaller than 4GB it is often
  74            larger than 16MB and in this case we have a chance of
  75            finding fitting memory in the next higher zone first. If
  76            not retry with true GFP_DMA. -AK */
  77         if (dma_mask <= DMA_32BIT_MASK)
  78                 gfp |= GFP_DMA32;
  79
  80  again:
  81         page = dma_alloc_pages(dev, gfp, get_order(size));
  82         if (page == NULL)
  83                 return NULL;
  84
  85         {
  86                 int high, mmu;
  87                 bus = page_to_phys(page);
  88                 memory = page_address(page);
  89                 high = (bus + size) >= dma_mask;
  90                 mmu = high;
  91                 if (force_iommu && !(gfp & GFP_DMA))
  92                         mmu = 1;
  93                 else if (high) {
  94                         free_pages((unsigned long)memory,
  95                                    get_order(size));
  96
  97                         /* Don't use the 16MB ZONE_DMA unless absolutely
  98                            needed. It's better to use remapping first. */
  99                         if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
 100                                 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
 101                                 goto again;
 102                         }
 103
 104                         /* Let low level make its own zone decisions */
 105                         gfp &= ~(GFP_DMA32|GFP_DMA);
 106
 107                         if (dma_ops->alloc_coherent)
 108                                 return dma_ops->alloc_coherent(dev, size,
 109                                                            dma_handle, gfp);
 110                         return NULL;
 111                 }
 112
 113                 memset(memory, 0, size);
 114                 if (!mmu) {
 115                         *dma_handle = bus;
 116                         return memory;
 117                 }
 118         }
 119
 120         if (dma_ops->alloc_coherent) {
 121                 free_pages((unsigned long)memory, get_order(size));
 122                 gfp &= ~(GFP_DMA|GFP_DMA32);
 123                 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
 124         }
 125
 126         if (dma_ops->map_simple) {
 127                 *dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
 128                                               size,
 129                                               PCI_DMA_BIDIRECTIONAL);
 130                 if (*dma_handle != bad_dma_address)
 131                         return memory;
 132         }
 133
 134         if (panic_on_overflow)
 135                 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
 136         free_pages((unsigned long)memory, get_order(size));
 137         return NULL;
 138 }
 139 EXPORT_SYMBOL(dma_alloc_coherent);
 140
 141 /*
 142  * Unmap coherent memory.
 143  * The caller must ensure that the device has finished accessing the mapping.
 144  */
 145 void dma_free_coherent(struct device *dev, size_t size,
 146                          void *vaddr, dma_addr_t bus)
 147 {
 148         int order = get_order(size);
 149         WARN_ON(irqs_disabled());       /* for portability */
 150         if (dma_release_coherent(dev, order, vaddr))
 151                 return;
 152         if (dma_ops->unmap_single)
 153                 dma_ops->unmap_single(dev, bus, size, 0);
 154         free_pages((unsigned long)vaddr, order);
 155 }
 156 EXPORT_SYMBOL(dma_free_coherent);