drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include "ttm/ttm_bo_driver.h"
  32 #include "ttm/ttm_placement.h"
  33 #include <linux/io.h>
  34 #include <linux/highmem.h>
  35 #include <linux/wait.h>
  36 #include <linux/vmalloc.h>
  37 #include <linux/module.h>
  38
  39 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  40 {
  41         struct ttm_mem_reg *old_mem = &bo->mem;
  42
  43         if (old_mem->mm_node) {
  44                 spin_lock(&bo->glob->lru_lock);
  45                 drm_mm_put_block(old_mem->mm_node);
  46                 spin_unlock(&bo->glob->lru_lock);
  47         }
  48         old_mem->mm_node = NULL;
  49 }
  50
  51 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  52                     bool evict, bool no_wait_reserve,
  53                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  54 {
  55         struct ttm_tt *ttm = bo->ttm;
  56         struct ttm_mem_reg *old_mem = &bo->mem;
  57         int ret;
  58
  59         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  60                 ttm_tt_unbind(ttm);
  61                 ttm_bo_free_old_node(bo);
  62                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  63                                 TTM_PL_MASK_MEM);
  64                 old_mem->mem_type = TTM_PL_SYSTEM;
  65         }
  66
  67         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  68         if (unlikely(ret != 0))
  69                 return ret;
  70
  71         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  72                 ret = ttm_tt_bind(ttm, new_mem);
  73                 if (unlikely(ret != 0))
  74                         return ret;
  75         }
  76
  77         *old_mem = *new_mem;
  78         new_mem->mm_node = NULL;
  79
  80         return 0;
  81 }
  82 EXPORT_SYMBOL(ttm_bo_move_ttm);
  83
  84 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  85 {
  86         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
  87         int ret;
  88
  89         if (bdev->driver->io_mem_reserve) {
  90                 if (!mem->bus.io_reserved) {
  91                         mem->bus.io_reserved = true;
  92                         ret = bdev->driver->io_mem_reserve(bdev, mem);
  93                         if (unlikely(ret != 0))
  94                                 return ret;
  95                 }
  96         } else {
  97                 ret = ttm_bo_pci_offset(bdev, mem, &mem->bus.base, &mem->bus.offset, &mem->bus.size);
  98                 if (unlikely(ret != 0))
  99                         return ret;
 100                 mem->bus.addr = NULL;
 101                 if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP))
 102                         mem->bus.addr = (void *)(((u8 *)man->io_addr) + mem->bus.offset);
 103                 mem->bus.is_iomem = (mem->bus.size > 0) ? 1 : 0;
 104         }
 105         return 0;
 106 }
 107
 108 void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 109 {
 110         if (bdev->driver->io_mem_reserve) {
 111                 if (mem->bus.io_reserved) {
 112                         mem->bus.io_reserved = false;
 113                         bdev->driver->io_mem_free(bdev, mem);
 114                 }
 115         }
 116 }
 117
 118 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 119                         void **virtual)
 120 {
 121         int ret;
 122         void *addr;
 123
 124         *virtual = NULL;
 125         ret = ttm_mem_io_reserve(bdev, mem);
 126         if (ret)
 127                 return ret;
 128
 129         if (mem->bus.addr) {
 130                 addr = mem->bus.addr;
 131         } else {
 132                 if (mem->placement & TTM_PL_FLAG_WC)
 133                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 134                 else
 135                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 136                 if (!addr) {
 137                         ttm_mem_io_free(bdev, mem);
 138                         return -ENOMEM;
 139                 }
 140         }
 141         *virtual = addr;
 142         return 0;
 143 }
 144
 145 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 146                          void *virtual)
 147 {
 148         struct ttm_mem_type_manager *man;
 149
 150         man = &bdev->man[mem->mem_type];
 151
 152         if (virtual && (man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP || mem->bus.addr == NULL))
 153                 iounmap(virtual);
 154         ttm_mem_io_free(bdev, mem);
 155 }
 156
 157 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 158 {
 159         uint32_t *dstP =
 160             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 161         uint32_t *srcP =
 162             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 163
 164         int i;
 165         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 166                 iowrite32(ioread32(srcP++), dstP++);
 167         return 0;
 168 }
 169
 170 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 171                                 unsigned long page,
 172                                 pgprot_t prot)
 173 {
 174         struct page *d = ttm_tt_get_page(ttm, page);
 175         void *dst;
 176
 177         if (!d)
 178                 return -ENOMEM;
 179
 180         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 181
 182 #ifdef CONFIG_X86
 183         dst = kmap_atomic_prot(d, KM_USER0, prot);
 184 #else
 185         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 186                 dst = vmap(&d, 1, 0, prot);
 187         else
 188                 dst = kmap(d);
 189 #endif
 190         if (!dst)
 191                 return -ENOMEM;
 192
 193         memcpy_fromio(dst, src, PAGE_SIZE);
 194
 195 #ifdef CONFIG_X86
 196         kunmap_atomic(dst, KM_USER0);
 197 #else
 198         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 199                 vunmap(dst);
 200         else
 201                 kunmap(d);
 202 #endif
 203
 204         return 0;
 205 }
 206
 207 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 208                                 unsigned long page,
 209                                 pgprot_t prot)
 210 {
 211         struct page *s = ttm_tt_get_page(ttm, page);
 212         void *src;
 213
 214         if (!s)
 215                 return -ENOMEM;
 216
 217         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 218 #ifdef CONFIG_X86
 219         src = kmap_atomic_prot(s, KM_USER0, prot);
 220 #else
 221         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 222                 src = vmap(&s, 1, 0, prot);
 223         else
 224                 src = kmap(s);
 225 #endif
 226         if (!src)
 227                 return -ENOMEM;
 228
 229         memcpy_toio(dst, src, PAGE_SIZE);
 230
 231 #ifdef CONFIG_X86
 232         kunmap_atomic(src, KM_USER0);
 233 #else
 234         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 235                 vunmap(src);
 236         else
 237                 kunmap(s);
 238 #endif
 239
 240         return 0;
 241 }
 242
 243 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 244                        bool evict, bool no_wait_reserve, bool no_wait_gpu,
 245                        struct ttm_mem_reg *new_mem)
 246 {
 247         struct ttm_bo_device *bdev = bo->bdev;
 248         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 249         struct ttm_tt *ttm = bo->ttm;
 250         struct ttm_mem_reg *old_mem = &bo->mem;
 251         struct ttm_mem_reg old_copy = *old_mem;
 252         void *old_iomap;
 253         void *new_iomap;
 254         int ret;
 255         unsigned long i;
 256         unsigned long page;
 257         unsigned long add = 0;
 258         int dir;
 259
 260         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 261         if (ret)
 262                 return ret;
 263         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 264         if (ret)
 265                 goto out;
 266
 267         if (old_iomap == NULL && new_iomap == NULL)
 268                 goto out2;
 269         if (old_iomap == NULL && ttm == NULL)
 270                 goto out2;
 271
 272         add = 0;
 273         dir = 1;
 274
 275         if ((old_mem->mem_type == new_mem->mem_type) &&
 276             (new_mem->mm_node->start <
 277              old_mem->mm_node->start + old_mem->mm_node->size)) {
 278                 dir = -1;
 279                 add = new_mem->num_pages - 1;
 280         }
 281
 282         for (i = 0; i < new_mem->num_pages; ++i) {
 283                 page = i * dir + add;
 284                 if (old_iomap == NULL) {
 285                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 286                                                     PAGE_KERNEL);
 287                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 288                                                    prot);
 289                 } else if (new_iomap == NULL) {
 290                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 291                                                     PAGE_KERNEL);
 292                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 293                                                    prot);
 294                 } else
 295                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 296                 if (ret)
 297                         goto out1;
 298         }
 299         mb();
 300 out2:
 301         ttm_bo_free_old_node(bo);
 302
 303         *old_mem = *new_mem;
 304         new_mem->mm_node = NULL;
 305
 306         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 307                 ttm_tt_unbind(ttm);
 308                 ttm_tt_destroy(ttm);
 309                 bo->ttm = NULL;
 310         }
 311
 312 out1:
 313         ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
 314 out:
 315         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 316         return ret;
 317 }
 318 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 319
 320 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 321 {
 322         kfree(bo);
 323 }
 324
 325 /**
 326  * ttm_buffer_object_transfer
 327  *
 328  * @bo: A pointer to a struct ttm_buffer_object.
 329  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 330  * holding the data of @bo with the old placement.
 331  *
 332  * This is a utility function that may be called after an accelerated move
 333  * has been scheduled. A new buffer object is created as a placeholder for
 334  * the old data while it's being copied. When that buffer object is idle,
 335  * it can be destroyed, releasing the space of the old placement.
 336  * Returns:
 337  * !0: Failure.
 338  */
 339
 340 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 341                                       struct ttm_buffer_object **new_obj)
 342 {
 343         struct ttm_buffer_object *fbo;
 344         struct ttm_bo_device *bdev = bo->bdev;
 345         struct ttm_bo_driver *driver = bdev->driver;
 346
 347         fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
 348         if (!fbo)
 349                 return -ENOMEM;
 350
 351         *fbo = *bo;
 352
 353         /**
 354          * Fix up members that we shouldn't copy directly:
 355          * TODO: Explicit member copy would probably be better here.
 356          */
 357
 358         spin_lock_init(&fbo->lock);
 359         init_waitqueue_head(&fbo->event_queue);
 360         INIT_LIST_HEAD(&fbo->ddestroy);
 361         INIT_LIST_HEAD(&fbo->lru);
 362         INIT_LIST_HEAD(&fbo->swap);
 363         fbo->vm_node = NULL;
 364
 365         fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 366         if (fbo->mem.mm_node)
 367                 fbo->mem.mm_node->private = (void *)fbo;
 368         kref_init(&fbo->list_kref);
 369         kref_init(&fbo->kref);
 370         fbo->destroy = &ttm_transfered_destroy;
 371
 372         *new_obj = fbo;
 373         return 0;
 374 }
 375
 376 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 377 {
 378 #if defined(__i386__) || defined(__x86_64__)
 379         if (caching_flags & TTM_PL_FLAG_WC)
 380                 tmp = pgprot_writecombine(tmp);
 381         else if (boot_cpu_data.x86 > 3)
 382                 tmp = pgprot_noncached(tmp);
 383
 384 #elif defined(__powerpc__)
 385         if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
 386                 pgprot_val(tmp) |= _PAGE_NO_CACHE;
 387                 if (caching_flags & TTM_PL_FLAG_UNCACHED)
 388                         pgprot_val(tmp) |= _PAGE_GUARDED;
 389         }
 390 #endif
 391 #if defined(__ia64__)
 392         if (caching_flags & TTM_PL_FLAG_WC)
 393                 tmp = pgprot_writecombine(tmp);
 394         else
 395                 tmp = pgprot_noncached(tmp);
 396 #endif
 397 #if defined(__sparc__)
 398         if (!(caching_flags & TTM_PL_FLAG_CACHED))
 399                 tmp = pgprot_noncached(tmp);
 400 #endif
 401         return tmp;
 402 }
 403 EXPORT_SYMBOL(ttm_io_prot);
 404
 405 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 406                           unsigned long offset,
 407                           unsigned long size,
 408                           struct ttm_bo_kmap_obj *map)
 409 {
 410         struct ttm_mem_reg *mem = &bo->mem;
 411
 412         if (bo->mem.bus.addr) {
 413                 map->bo_kmap_type = ttm_bo_map_premapped;
 414                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 415         } else {
 416                 map->bo_kmap_type = ttm_bo_map_iomap;
 417                 if (mem->placement & TTM_PL_FLAG_WC)
 418                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 419                                                   size);
 420                 else
 421                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 422                                                        size);
 423         }
 424         return (!map->virtual) ? -ENOMEM : 0;
 425 }
 426
 427 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 428                            unsigned long start_page,
 429                            unsigned long num_pages,
 430                            struct ttm_bo_kmap_obj *map)
 431 {
 432         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
 433         struct ttm_tt *ttm = bo->ttm;
 434         struct page *d;
 435         int i;
 436
 437         BUG_ON(!ttm);
 438         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 439                 /*
 440                  * We're mapping a single page, and the desired
 441                  * page protection is consistent with the bo.
 442                  */
 443
 444                 map->bo_kmap_type = ttm_bo_map_kmap;
 445                 map->page = ttm_tt_get_page(ttm, start_page);
 446                 map->virtual = kmap(map->page);
 447         } else {
 448             /*
 449              * Populate the part we're mapping;
 450              */
 451                 for (i = start_page; i < start_page + num_pages; ++i) {
 452                         d = ttm_tt_get_page(ttm, i);
 453                         if (!d)
 454                                 return -ENOMEM;
 455                 }
 456
 457                 /*
 458                  * We need to use vmap to get the desired page protection
 459                  * or to make the buffer object look contiguous.
 460                  */
 461                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 462                         PAGE_KERNEL :
 463                         ttm_io_prot(mem->placement, PAGE_KERNEL);
 464                 map->bo_kmap_type = ttm_bo_map_vmap;
 465                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 466                                     0, prot);
 467         }
 468         return (!map->virtual) ? -ENOMEM : 0;
 469 }
 470
 471 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 472                 unsigned long start_page, unsigned long num_pages,
 473                 struct ttm_bo_kmap_obj *map)
 474 {
 475         unsigned long offset, size;
 476         int ret;
 477
 478         BUG_ON(!list_empty(&bo->swap));
 479         map->virtual = NULL;
 480         map->bo = bo;
 481         if (num_pages > bo->num_pages)
 482                 return -EINVAL;
 483         if (start_page > bo->num_pages)
 484                 return -EINVAL;
 485 #if 0
 486         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 487                 return -EPERM;
 488 #endif
 489         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 490         if (ret)
 491                 return ret;
 492         if (!bo->mem.bus.is_iomem) {
 493                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 494         } else {
 495                 offset = start_page << PAGE_SHIFT;
 496                 size = num_pages << PAGE_SHIFT;
 497                 return ttm_bo_ioremap(bo, offset, size, map);
 498         }
 499 }
 500 EXPORT_SYMBOL(ttm_bo_kmap);
 501
 502 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 503 {
 504         if (!map->virtual)
 505                 return;
 506         switch (map->bo_kmap_type) {
 507         case ttm_bo_map_iomap:
 508                 iounmap(map->virtual);
 509                 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 510                 break;
 511         case ttm_bo_map_vmap:
 512                 vunmap(map->virtual);
 513                 break;
 514         case ttm_bo_map_kmap:
 515                 kunmap(map->page);
 516                 break;
 517         case ttm_bo_map_premapped:
 518                 break;
 519         default:
 520                 BUG();
 521         }
 522         map->virtual = NULL;
 523         map->page = NULL;
 524 }
 525 EXPORT_SYMBOL(ttm_bo_kunmap);
 526
 527 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 528                               void *sync_obj,
 529                               void *sync_obj_arg,
 530                               bool evict, bool no_wait_reserve,
 531                               bool no_wait_gpu,
 532                               struct ttm_mem_reg *new_mem)
 533 {
 534         struct ttm_bo_device *bdev = bo->bdev;
 535         struct ttm_bo_driver *driver = bdev->driver;
 536         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 537         struct ttm_mem_reg *old_mem = &bo->mem;
 538         int ret;
 539         struct ttm_buffer_object *ghost_obj;
 540         void *tmp_obj = NULL;
 541
 542         spin_lock(&bo->lock);
 543         if (bo->sync_obj) {
 544                 tmp_obj = bo->sync_obj;
 545                 bo->sync_obj = NULL;
 546         }
 547         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 548         bo->sync_obj_arg = sync_obj_arg;
 549         if (evict) {
 550                 ret = ttm_bo_wait(bo, false, false, false);
 551                 spin_unlock(&bo->lock);
 552                 if (tmp_obj)
 553                         driver->sync_obj_unref(&tmp_obj);
 554                 if (ret)
 555                         return ret;
 556
 557                 ttm_bo_free_old_node(bo);
 558                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 559                     (bo->ttm != NULL)) {
 560                         ttm_tt_unbind(bo->ttm);
 561                         ttm_tt_destroy(bo->ttm);
 562                         bo->ttm = NULL;
 563                 }
 564         } else {
 565                 /**
 566                  * This should help pipeline ordinary buffer moves.
 567                  *
 568                  * Hang old buffer memory on a new buffer object,
 569                  * and leave it to be released when the GPU
 570                  * operation has completed.
 571                  */
 572
 573                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 574                 spin_unlock(&bo->lock);
 575                 if (tmp_obj)
 576                         driver->sync_obj_unref(&tmp_obj);
 577
 578                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 579                 if (ret)
 580                         return ret;
 581
 582                 /**
 583                  * If we're not moving to fixed memory, the TTM object
 584                  * needs to stay alive. Otherwhise hang it on the ghost
 585                  * bo to be unbound and destroyed.
 586                  */
 587
 588                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 589                         ghost_obj->ttm = NULL;
 590                 else
 591                         bo->ttm = NULL;
 592
 593                 ttm_bo_unreserve(ghost_obj);
 594                 ttm_bo_unref(&ghost_obj);
 595         }
 596
 597         *old_mem = *new_mem;
 598         new_mem->mm_node = NULL;
 599
 600         return 0;
 601 }
 602 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);