drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "drmP.h"
  26 #include "drm.h"
  27 #include "i915_drm.h"
  28 #include "i915_drv.h"
  29 #include "i915_trace.h"
  30 #include "intel_drv.h"
  31
  32 /* PPGTT support for Sandybdrige/Gen6 and later */
  33 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  34                                    unsigned first_entry,
  35                                    unsigned num_entries)
  36 {
  37         uint32_t *pt_vaddr;
  38         uint32_t scratch_pte;
  39         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  40         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  41         unsigned last_pte, i;
  42
  43         scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
  44         scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
  45
  46         while (num_entries) {
  47                 last_pte = first_pte + num_entries;
  48                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  49                         last_pte = I915_PPGTT_PT_ENTRIES;
  50
  51                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
  52
  53                 for (i = first_pte; i < last_pte; i++)
  54                         pt_vaddr[i] = scratch_pte;
  55
  56                 kunmap_atomic(pt_vaddr);
  57
  58                 num_entries -= last_pte - first_pte;
  59                 first_pte = 0;
  60                 act_pd++;
  61         }
  62 }
  63
  64 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  65 {
  66         struct drm_i915_private *dev_priv = dev->dev_private;
  67         struct i915_hw_ppgtt *ppgtt;
  68         unsigned first_pd_entry_in_global_pt;
  69         int i;
  70         int ret = -ENOMEM;
  71
  72         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  73          * entries. For aliasing ppgtt support we just steal them at the end for
  74          * now. */
  75         first_pd_entry_in_global_pt = 512*1024 - I915_PPGTT_PD_ENTRIES;
  76
  77         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  78         if (!ppgtt)
  79                 return ret;
  80
  81         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  82         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  83                                   GFP_KERNEL);
  84         if (!ppgtt->pt_pages)
  85                 goto err_ppgtt;
  86
  87         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  88                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  89                 if (!ppgtt->pt_pages[i])
  90                         goto err_pt_alloc;
  91         }
  92
  93         if (dev_priv->mm.gtt->needs_dmar) {
  94                 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
  95                                                 *ppgtt->num_pd_entries,
  96                                              GFP_KERNEL);
  97                 if (!ppgtt->pt_dma_addr)
  98                         goto err_pt_alloc;
  99
 100                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
 101                         dma_addr_t pt_addr;
 102
 103                         pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
 104                                                0, 4096,
 105                                                PCI_DMA_BIDIRECTIONAL);
 106
 107                         if (pci_dma_mapping_error(dev->pdev,
 108                                                   pt_addr)) {
 109                                 ret = -EIO;
 110                                 goto err_pd_pin;
 111
 112                         }
 113                         ppgtt->pt_dma_addr[i] = pt_addr;
 114                 }
 115         }
 116
 117         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
 118
 119         i915_ppgtt_clear_range(ppgtt, 0,
 120                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 121
 122         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t);
 123
 124         dev_priv->mm.aliasing_ppgtt = ppgtt;
 125
 126         return 0;
 127
 128 err_pd_pin:
 129         if (ppgtt->pt_dma_addr) {
 130                 for (i--; i >= 0; i--)
 131                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 132                                        4096, PCI_DMA_BIDIRECTIONAL);
 133         }
 134 err_pt_alloc:
 135         kfree(ppgtt->pt_dma_addr);
 136         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 137                 if (ppgtt->pt_pages[i])
 138                         __free_page(ppgtt->pt_pages[i]);
 139         }
 140         kfree(ppgtt->pt_pages);
 141 err_ppgtt:
 142         kfree(ppgtt);
 143
 144         return ret;
 145 }
 146
 147 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 148 {
 149         struct drm_i915_private *dev_priv = dev->dev_private;
 150         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 151         int i;
 152
 153         if (!ppgtt)
 154                 return;
 155
 156         if (ppgtt->pt_dma_addr) {
 157                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 158                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 159                                        4096, PCI_DMA_BIDIRECTIONAL);
 160         }
 161
 162         kfree(ppgtt->pt_dma_addr);
 163         for (i = 0; i < ppgtt->num_pd_entries; i++)
 164                 __free_page(ppgtt->pt_pages[i]);
 165         kfree(ppgtt->pt_pages);
 166         kfree(ppgtt);
 167 }
 168
 169 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 170                                          struct scatterlist *sg_list,
 171                                          unsigned sg_len,
 172                                          unsigned first_entry,
 173                                          uint32_t pte_flags)
 174 {
 175         uint32_t *pt_vaddr, pte;
 176         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 177         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 178         unsigned i, j, m, segment_len;
 179         dma_addr_t page_addr;
 180         struct scatterlist *sg;
 181
 182         /* init sg walking */
 183         sg = sg_list;
 184         i = 0;
 185         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 186         m = 0;
 187
 188         while (i < sg_len) {
 189                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 190
 191                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
 192                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 193                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 194                         pt_vaddr[j] = pte | pte_flags;
 195
 196                         /* grab the next page */
 197                         m++;
 198                         if (m == segment_len) {
 199                                 sg = sg_next(sg);
 200                                 i++;
 201                                 if (i == sg_len)
 202                                         break;
 203
 204                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 205                                 m = 0;
 206                         }
 207                 }
 208
 209                 kunmap_atomic(pt_vaddr);
 210
 211                 first_pte = 0;
 212                 act_pd++;
 213         }
 214 }
 215
 216 static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt,
 217                                     unsigned first_entry, unsigned num_entries,
 218                                     struct page **pages, uint32_t pte_flags)
 219 {
 220         uint32_t *pt_vaddr, pte;
 221         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 222         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 223         unsigned last_pte, i;
 224         dma_addr_t page_addr;
 225
 226         while (num_entries) {
 227                 last_pte = first_pte + num_entries;
 228                 last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES);
 229
 230                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 231
 232                 for (i = first_pte; i < last_pte; i++) {
 233                         page_addr = page_to_phys(*pages);
 234                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 235                         pt_vaddr[i] = pte | pte_flags;
 236
 237                         pages++;
 238                 }
 239
 240                 kunmap_atomic(pt_vaddr);
 241
 242                 num_entries -= last_pte - first_pte;
 243                 first_pte = 0;
 244                 act_pd++;
 245         }
 246 }
 247
 248 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 249                             struct drm_i915_gem_object *obj,
 250                             enum i915_cache_level cache_level)
 251 {
 252         struct drm_device *dev = obj->base.dev;
 253         struct drm_i915_private *dev_priv = dev->dev_private;
 254         uint32_t pte_flags = GEN6_PTE_VALID;
 255
 256         switch (cache_level) {
 257         case I915_CACHE_LLC_MLC:
 258                 pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
 259                 break;
 260         case I915_CACHE_LLC:
 261                 pte_flags |= GEN6_PTE_CACHE_LLC;
 262                 break;
 263         case I915_CACHE_NONE:
 264                 pte_flags |= GEN6_PTE_UNCACHED;
 265                 break;
 266         default:
 267                 BUG();
 268         }
 269
 270         if (obj->sg_table) {
 271                 i915_ppgtt_insert_sg_entries(ppgtt,
 272                                              obj->sg_table->sgl,
 273                                              obj->sg_table->nents,
 274                                              obj->gtt_space->start >> PAGE_SHIFT,
 275                                              pte_flags);
 276         } else if (dev_priv->mm.gtt->needs_dmar) {
 277                 BUG_ON(!obj->sg_list);
 278
 279                 i915_ppgtt_insert_sg_entries(ppgtt,
 280                                              obj->sg_list,
 281                                              obj->num_sg,
 282                                              obj->gtt_space->start >> PAGE_SHIFT,
 283                                              pte_flags);
 284         } else
 285                 i915_ppgtt_insert_pages(ppgtt,
 286                                         obj->gtt_space->start >> PAGE_SHIFT,
 287                                         obj->base.size >> PAGE_SHIFT,
 288                                         obj->pages,
 289                                         pte_flags);
 290 }
 291
 292 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 293                               struct drm_i915_gem_object *obj)
 294 {
 295         i915_ppgtt_clear_range(ppgtt,
 296                                obj->gtt_space->start >> PAGE_SHIFT,
 297                                obj->base.size >> PAGE_SHIFT);
 298 }
 299
 300 /* XXX kill agp_type! */
 301 static unsigned int cache_level_to_agp_type(struct drm_device *dev,
 302                                             enum i915_cache_level cache_level)
 303 {
 304         switch (cache_level) {
 305         case I915_CACHE_LLC_MLC:
 306                 if (INTEL_INFO(dev)->gen >= 6)
 307                         return AGP_USER_CACHED_MEMORY_LLC_MLC;
 308                 /* Older chipsets do not have this extra level of CPU
 309                  * cacheing, so fallthrough and request the PTE simply
 310                  * as cached.
 311                  */
 312         case I915_CACHE_LLC:
 313                 return AGP_USER_CACHED_MEMORY;
 314         default:
 315         case I915_CACHE_NONE:
 316                 return AGP_USER_MEMORY;
 317         }
 318 }
 319
 320 static bool do_idling(struct drm_i915_private *dev_priv)
 321 {
 322         bool ret = dev_priv->mm.interruptible;
 323
 324         if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
 325                 dev_priv->mm.interruptible = false;
 326                 if (i915_gpu_idle(dev_priv->dev)) {
 327                         DRM_ERROR("Couldn't idle GPU\n");
 328                         /* Wait a bit, in hopes it avoids the hang */
 329                         udelay(10);
 330                 }
 331         }
 332
 333         return ret;
 334 }
 335
 336 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 337 {
 338         if (unlikely(dev_priv->mm.gtt->do_idle_maps))
 339                 dev_priv->mm.interruptible = interruptible;
 340 }
 341
 342 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 343 {
 344         struct drm_i915_private *dev_priv = dev->dev_private;
 345         struct drm_i915_gem_object *obj;
 346
 347         /* First fill our portion of the GTT with scratch pages */
 348         intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 349                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 350
 351         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 352                 i915_gem_clflush_object(obj);
 353                 i915_gem_gtt_bind_object(obj, obj->cache_level);
 354         }
 355
 356         intel_gtt_chipset_flush();
 357 }
 358
 359 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 360 {
 361         struct drm_device *dev = obj->base.dev;
 362         struct drm_i915_private *dev_priv = dev->dev_private;
 363
 364         /* don't map imported dma buf objects */
 365         if (dev_priv->mm.gtt->needs_dmar && !obj->sg_table)
 366                 return intel_gtt_map_memory(obj->pages,
 367                                             obj->base.size >> PAGE_SHIFT,
 368                                             &obj->sg_list,
 369                                             &obj->num_sg);
 370         else
 371                 return 0;
 372 }
 373
 374 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 375                               enum i915_cache_level cache_level)
 376 {
 377         struct drm_device *dev = obj->base.dev;
 378         struct drm_i915_private *dev_priv = dev->dev_private;
 379         unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
 380
 381         if (obj->sg_table) {
 382                 intel_gtt_insert_sg_entries(obj->sg_table->sgl,
 383                                             obj->sg_table->nents,
 384                                             obj->gtt_space->start >> PAGE_SHIFT,
 385                                             agp_type);
 386         } else if (dev_priv->mm.gtt->needs_dmar) {
 387                 BUG_ON(!obj->sg_list);
 388
 389                 intel_gtt_insert_sg_entries(obj->sg_list,
 390                                             obj->num_sg,
 391                                             obj->gtt_space->start >> PAGE_SHIFT,
 392                                             agp_type);
 393         } else
 394                 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 395                                        obj->base.size >> PAGE_SHIFT,
 396                                        obj->pages,
 397                                        agp_type);
 398
 399         obj->has_global_gtt_mapping = 1;
 400 }
 401
 402 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 403 {
 404         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
 405                               obj->base.size >> PAGE_SHIFT);
 406
 407         obj->has_global_gtt_mapping = 0;
 408 }
 409
 410 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 411 {
 412         struct drm_device *dev = obj->base.dev;
 413         struct drm_i915_private *dev_priv = dev->dev_private;
 414         bool interruptible;
 415
 416         interruptible = do_idling(dev_priv);
 417
 418         if (obj->sg_list) {
 419                 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
 420                 obj->sg_list = NULL;
 421         }
 422
 423         undo_idling(dev_priv, interruptible);
 424 }
 425
 426 void i915_gem_init_global_gtt(struct drm_device *dev,
 427                               unsigned long start,
 428                               unsigned long mappable_end,
 429                               unsigned long end)
 430 {
 431         drm_i915_private_t *dev_priv = dev->dev_private;
 432
 433         /* Substract the guard page ... */
 434         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
 435
 436         dev_priv->mm.gtt_start = start;
 437         dev_priv->mm.gtt_mappable_end = mappable_end;
 438         dev_priv->mm.gtt_end = end;
 439         dev_priv->mm.gtt_total = end - start;
 440         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 441
 442         /* ... but ensure that we clear the entire range. */
 443         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 444 }