drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drmP.h>
  26 #include <drm/i915_drm.h>
  27 #include "i915_drv.h"
  28 #include "i915_trace.h"
  29 #include "intel_drv.h"
  30
  31 /* PPGTT support for Sandybdrige/Gen6 and later */
  32 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  33                                    unsigned first_entry,
  34                                    unsigned num_entries)
  35 {
  36         uint32_t *pt_vaddr;
  37         uint32_t scratch_pte;
  38         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  39         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  40         unsigned last_pte, i;
  41
  42         scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
  43         scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
  44
  45         while (num_entries) {
  46                 last_pte = first_pte + num_entries;
  47                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  48                         last_pte = I915_PPGTT_PT_ENTRIES;
  49
  50                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
  51
  52                 for (i = first_pte; i < last_pte; i++)
  53                         pt_vaddr[i] = scratch_pte;
  54
  55                 kunmap_atomic(pt_vaddr);
  56
  57                 num_entries -= last_pte - first_pte;
  58                 first_pte = 0;
  59                 act_pd++;
  60         }
  61 }
  62
  63 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  64 {
  65         struct drm_i915_private *dev_priv = dev->dev_private;
  66         struct i915_hw_ppgtt *ppgtt;
  67         unsigned first_pd_entry_in_global_pt;
  68         int i;
  69         int ret = -ENOMEM;
  70
  71         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  72          * entries. For aliasing ppgtt support we just steal them at the end for
  73          * now. */
  74         first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES;
  75
  76         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  77         if (!ppgtt)
  78                 return ret;
  79
  80         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  81         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  82                                   GFP_KERNEL);
  83         if (!ppgtt->pt_pages)
  84                 goto err_ppgtt;
  85
  86         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  87                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  88                 if (!ppgtt->pt_pages[i])
  89                         goto err_pt_alloc;
  90         }
  91
  92         if (dev_priv->mm.gtt->needs_dmar) {
  93                 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
  94                                                 *ppgtt->num_pd_entries,
  95                                              GFP_KERNEL);
  96                 if (!ppgtt->pt_dma_addr)
  97                         goto err_pt_alloc;
  98
  99                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
 100                         dma_addr_t pt_addr;
 101
 102                         pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
 103                                                0, 4096,
 104                                                PCI_DMA_BIDIRECTIONAL);
 105
 106                         if (pci_dma_mapping_error(dev->pdev,
 107                                                   pt_addr)) {
 108                                 ret = -EIO;
 109                                 goto err_pd_pin;
 110
 111                         }
 112                         ppgtt->pt_dma_addr[i] = pt_addr;
 113                 }
 114         }
 115
 116         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
 117
 118         i915_ppgtt_clear_range(ppgtt, 0,
 119                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 120
 121         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t);
 122
 123         dev_priv->mm.aliasing_ppgtt = ppgtt;
 124
 125         return 0;
 126
 127 err_pd_pin:
 128         if (ppgtt->pt_dma_addr) {
 129                 for (i--; i >= 0; i--)
 130                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 131                                        4096, PCI_DMA_BIDIRECTIONAL);
 132         }
 133 err_pt_alloc:
 134         kfree(ppgtt->pt_dma_addr);
 135         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 136                 if (ppgtt->pt_pages[i])
 137                         __free_page(ppgtt->pt_pages[i]);
 138         }
 139         kfree(ppgtt->pt_pages);
 140 err_ppgtt:
 141         kfree(ppgtt);
 142
 143         return ret;
 144 }
 145
 146 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 147 {
 148         struct drm_i915_private *dev_priv = dev->dev_private;
 149         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 150         int i;
 151
 152         if (!ppgtt)
 153                 return;
 154
 155         if (ppgtt->pt_dma_addr) {
 156                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 157                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 158                                        4096, PCI_DMA_BIDIRECTIONAL);
 159         }
 160
 161         kfree(ppgtt->pt_dma_addr);
 162         for (i = 0; i < ppgtt->num_pd_entries; i++)
 163                 __free_page(ppgtt->pt_pages[i]);
 164         kfree(ppgtt->pt_pages);
 165         kfree(ppgtt);
 166 }
 167
 168 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 169                                          const struct sg_table *pages,
 170                                          unsigned first_entry,
 171                                          uint32_t pte_flags)
 172 {
 173         uint32_t *pt_vaddr, pte;
 174         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 175         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 176         unsigned i, j, m, segment_len;
 177         dma_addr_t page_addr;
 178         struct scatterlist *sg;
 179
 180         /* init sg walking */
 181         sg = pages->sgl;
 182         i = 0;
 183         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 184         m = 0;
 185
 186         while (i < pages->nents) {
 187                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 188
 189                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
 190                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 191                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 192                         pt_vaddr[j] = pte | pte_flags;
 193
 194                         /* grab the next page */
 195                         if (++m == segment_len) {
 196                                 if (++i == pages->nents)
 197                                         break;
 198
 199                                 sg = sg_next(sg);
 200                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 201                                 m = 0;
 202                         }
 203                 }
 204
 205                 kunmap_atomic(pt_vaddr);
 206
 207                 first_pte = 0;
 208                 act_pd++;
 209         }
 210 }
 211
 212 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 213                             struct drm_i915_gem_object *obj,
 214                             enum i915_cache_level cache_level)
 215 {
 216         uint32_t pte_flags = GEN6_PTE_VALID;
 217
 218         switch (cache_level) {
 219         case I915_CACHE_LLC_MLC:
 220                 pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
 221                 break;
 222         case I915_CACHE_LLC:
 223                 pte_flags |= GEN6_PTE_CACHE_LLC;
 224                 break;
 225         case I915_CACHE_NONE:
 226                 if (IS_HASWELL(obj->base.dev))
 227                         pte_flags |= HSW_PTE_UNCACHED;
 228                 else
 229                         pte_flags |= GEN6_PTE_UNCACHED;
 230                 break;
 231         default:
 232                 BUG();
 233         }
 234
 235         i915_ppgtt_insert_sg_entries(ppgtt,
 236                                      obj->pages,
 237                                      obj->gtt_space->start >> PAGE_SHIFT,
 238                                      pte_flags);
 239 }
 240
 241 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 242                               struct drm_i915_gem_object *obj)
 243 {
 244         i915_ppgtt_clear_range(ppgtt,
 245                                obj->gtt_space->start >> PAGE_SHIFT,
 246                                obj->base.size >> PAGE_SHIFT);
 247 }
 248
 249 /* XXX kill agp_type! */
 250 static unsigned int cache_level_to_agp_type(struct drm_device *dev,
 251                                             enum i915_cache_level cache_level)
 252 {
 253         switch (cache_level) {
 254         case I915_CACHE_LLC_MLC:
 255                 if (INTEL_INFO(dev)->gen >= 6)
 256                         return AGP_USER_CACHED_MEMORY_LLC_MLC;
 257                 /* Older chipsets do not have this extra level of CPU
 258                  * cacheing, so fallthrough and request the PTE simply
 259                  * as cached.
 260                  */
 261         case I915_CACHE_LLC:
 262                 return AGP_USER_CACHED_MEMORY;
 263         default:
 264         case I915_CACHE_NONE:
 265                 return AGP_USER_MEMORY;
 266         }
 267 }
 268
 269 static bool do_idling(struct drm_i915_private *dev_priv)
 270 {
 271         bool ret = dev_priv->mm.interruptible;
 272
 273         if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
 274                 dev_priv->mm.interruptible = false;
 275                 if (i915_gpu_idle(dev_priv->dev)) {
 276                         DRM_ERROR("Couldn't idle GPU\n");
 277                         /* Wait a bit, in hopes it avoids the hang */
 278                         udelay(10);
 279                 }
 280         }
 281
 282         return ret;
 283 }
 284
 285 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 286 {
 287         if (unlikely(dev_priv->mm.gtt->do_idle_maps))
 288                 dev_priv->mm.interruptible = interruptible;
 289 }
 290
 291 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 292 {
 293         struct drm_i915_private *dev_priv = dev->dev_private;
 294         struct drm_i915_gem_object *obj;
 295
 296         /* First fill our portion of the GTT with scratch pages */
 297         intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 298                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 299
 300         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
 301                 i915_gem_clflush_object(obj);
 302                 i915_gem_gtt_bind_object(obj, obj->cache_level);
 303         }
 304
 305         intel_gtt_chipset_flush();
 306 }
 307
 308 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 309 {
 310         if (obj->has_dma_mapping)
 311                 return 0;
 312
 313         if (!dma_map_sg(&obj->base.dev->pdev->dev,
 314                         obj->pages->sgl, obj->pages->nents,
 315                         PCI_DMA_BIDIRECTIONAL))
 316                 return -ENOSPC;
 317
 318         return 0;
 319 }
 320
 321 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 322                               enum i915_cache_level cache_level)
 323 {
 324         struct drm_device *dev = obj->base.dev;
 325         unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
 326
 327         intel_gtt_insert_sg_entries(obj->pages,
 328                                     obj->gtt_space->start >> PAGE_SHIFT,
 329                                     agp_type);
 330         obj->has_global_gtt_mapping = 1;
 331 }
 332
 333 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 334 {
 335         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
 336                               obj->base.size >> PAGE_SHIFT);
 337
 338         obj->has_global_gtt_mapping = 0;
 339 }
 340
 341 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 342 {
 343         struct drm_device *dev = obj->base.dev;
 344         struct drm_i915_private *dev_priv = dev->dev_private;
 345         bool interruptible;
 346
 347         interruptible = do_idling(dev_priv);
 348
 349         if (!obj->has_dma_mapping)
 350                 dma_unmap_sg(&dev->pdev->dev,
 351                              obj->pages->sgl, obj->pages->nents,
 352                              PCI_DMA_BIDIRECTIONAL);
 353
 354         undo_idling(dev_priv, interruptible);
 355 }
 356
 357 static void i915_gtt_color_adjust(struct drm_mm_node *node,
 358                                   unsigned long color,
 359                                   unsigned long *start,
 360                                   unsigned long *end)
 361 {
 362         if (node->color != color)
 363                 *start += 4096;
 364
 365         if (!list_empty(&node->node_list)) {
 366                 node = list_entry(node->node_list.next,
 367                                   struct drm_mm_node,
 368                                   node_list);
 369                 if (node->allocated && node->color != color)
 370                         *end -= 4096;
 371         }
 372 }
 373
 374 void i915_gem_init_global_gtt(struct drm_device *dev,
 375                               unsigned long start,
 376                               unsigned long mappable_end,
 377                               unsigned long end)
 378 {
 379         drm_i915_private_t *dev_priv = dev->dev_private;
 380
 381         /* Substract the guard page ... */
 382         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
 383         if (!HAS_LLC(dev))
 384                 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
 385
 386         dev_priv->mm.gtt_start = start;
 387         dev_priv->mm.gtt_mappable_end = mappable_end;
 388         dev_priv->mm.gtt_end = end;
 389         dev_priv->mm.gtt_total = end - start;
 390         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 391
 392         /* ... but ensure that we clear the entire range. */
 393         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 394 }