drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include "ttm/ttm_bo_driver.h"
  32 #include "ttm/ttm_placement.h"
  33 #include <linux/io.h>
  34 #include <linux/highmem.h>
  35 #include <linux/wait.h>
  36 #include <linux/slab.h>
  37 #include <linux/vmalloc.h>
  38 #include <linux/module.h>
  39
  40 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  41 {
  42         struct ttm_mem_reg *old_mem = &bo->mem;
  43
  44         if (old_mem->mm_node) {
  45                 spin_lock(&bo->glob->lru_lock);
  46                 drm_mm_put_block(old_mem->mm_node);
  47                 spin_unlock(&bo->glob->lru_lock);
  48         }
  49         old_mem->mm_node = NULL;
  50 }
  51
  52 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  53                     bool evict, bool no_wait_reserve,
  54                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  55 {
  56         struct ttm_tt *ttm = bo->ttm;
  57         struct ttm_mem_reg *old_mem = &bo->mem;
  58         int ret;
  59
  60         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  61                 ttm_tt_unbind(ttm);
  62                 ttm_bo_free_old_node(bo);
  63                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  64                                 TTM_PL_MASK_MEM);
  65                 old_mem->mem_type = TTM_PL_SYSTEM;
  66         }
  67
  68         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  69         if (unlikely(ret != 0))
  70                 return ret;
  71
  72         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  73                 ret = ttm_tt_bind(ttm, new_mem);
  74                 if (unlikely(ret != 0))
  75                         return ret;
  76         }
  77
  78         *old_mem = *new_mem;
  79         new_mem->mm_node = NULL;
  80
  81         return 0;
  82 }
  83 EXPORT_SYMBOL(ttm_bo_move_ttm);
  84
  85 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  86 {
  87         int ret;
  88
  89         if (!mem->bus.io_reserved) {
  90                 mem->bus.io_reserved = true;
  91                 ret = bdev->driver->io_mem_reserve(bdev, mem);
  92                 if (unlikely(ret != 0))
  93                         return ret;
  94         }
  95         return 0;
  96 }
  97
  98 void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  99 {
 100         if (bdev->driver->io_mem_reserve) {
 101                 if (mem->bus.io_reserved) {
 102                         mem->bus.io_reserved = false;
 103                         bdev->driver->io_mem_free(bdev, mem);
 104                 }
 105         }
 106 }
 107
 108 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 109                         void **virtual)
 110 {
 111         int ret;
 112         void *addr;
 113
 114         *virtual = NULL;
 115         ret = ttm_mem_io_reserve(bdev, mem);
 116         if (ret || !mem->bus.is_iomem)
 117                 return ret;
 118
 119         if (mem->bus.addr) {
 120                 addr = mem->bus.addr;
 121         } else {
 122                 if (mem->placement & TTM_PL_FLAG_WC)
 123                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 124                 else
 125                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 126                 if (!addr) {
 127                         ttm_mem_io_free(bdev, mem);
 128                         return -ENOMEM;
 129                 }
 130         }
 131         *virtual = addr;
 132         return 0;
 133 }
 134
 135 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 136                          void *virtual)
 137 {
 138         struct ttm_mem_type_manager *man;
 139
 140         man = &bdev->man[mem->mem_type];
 141
 142         if (virtual && mem->bus.addr == NULL)
 143                 iounmap(virtual);
 144         ttm_mem_io_free(bdev, mem);
 145 }
 146
 147 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 148 {
 149         uint32_t *dstP =
 150             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 151         uint32_t *srcP =
 152             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 153
 154         int i;
 155         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 156                 iowrite32(ioread32(srcP++), dstP++);
 157         return 0;
 158 }
 159
 160 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 161                                 unsigned long page,
 162                                 pgprot_t prot)
 163 {
 164         struct page *d = ttm_tt_get_page(ttm, page);
 165         void *dst;
 166
 167         if (!d)
 168                 return -ENOMEM;
 169
 170         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 171
 172 #ifdef CONFIG_X86
 173         dst = kmap_atomic_prot(d, KM_USER0, prot);
 174 #else
 175         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 176                 dst = vmap(&d, 1, 0, prot);
 177         else
 178                 dst = kmap(d);
 179 #endif
 180         if (!dst)
 181                 return -ENOMEM;
 182
 183         memcpy_fromio(dst, src, PAGE_SIZE);
 184
 185 #ifdef CONFIG_X86
 186         kunmap_atomic(dst, KM_USER0);
 187 #else
 188         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 189                 vunmap(dst);
 190         else
 191                 kunmap(d);
 192 #endif
 193
 194         return 0;
 195 }
 196
 197 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 198                                 unsigned long page,
 199                                 pgprot_t prot)
 200 {
 201         struct page *s = ttm_tt_get_page(ttm, page);
 202         void *src;
 203
 204         if (!s)
 205                 return -ENOMEM;
 206
 207         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 208 #ifdef CONFIG_X86
 209         src = kmap_atomic_prot(s, KM_USER0, prot);
 210 #else
 211         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 212                 src = vmap(&s, 1, 0, prot);
 213         else
 214                 src = kmap(s);
 215 #endif
 216         if (!src)
 217                 return -ENOMEM;
 218
 219         memcpy_toio(dst, src, PAGE_SIZE);
 220
 221 #ifdef CONFIG_X86
 222         kunmap_atomic(src, KM_USER0);
 223 #else
 224         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 225                 vunmap(src);
 226         else
 227                 kunmap(s);
 228 #endif
 229
 230         return 0;
 231 }
 232
 233 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 234                        bool evict, bool no_wait_reserve, bool no_wait_gpu,
 235                        struct ttm_mem_reg *new_mem)
 236 {
 237         struct ttm_bo_device *bdev = bo->bdev;
 238         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 239         struct ttm_tt *ttm = bo->ttm;
 240         struct ttm_mem_reg *old_mem = &bo->mem;
 241         struct ttm_mem_reg old_copy = *old_mem;
 242         void *old_iomap;
 243         void *new_iomap;
 244         int ret;
 245         unsigned long i;
 246         unsigned long page;
 247         unsigned long add = 0;
 248         int dir;
 249
 250         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 251         if (ret)
 252                 return ret;
 253         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 254         if (ret)
 255                 goto out;
 256
 257         if (old_iomap == NULL && new_iomap == NULL)
 258                 goto out2;
 259         if (old_iomap == NULL && ttm == NULL)
 260                 goto out2;
 261
 262         add = 0;
 263         dir = 1;
 264
 265         if ((old_mem->mem_type == new_mem->mem_type) &&
 266             (new_mem->mm_node->start <
 267              old_mem->mm_node->start + old_mem->mm_node->size)) {
 268                 dir = -1;
 269                 add = new_mem->num_pages - 1;
 270         }
 271
 272         for (i = 0; i < new_mem->num_pages; ++i) {
 273                 page = i * dir + add;
 274                 if (old_iomap == NULL) {
 275                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 276                                                     PAGE_KERNEL);
 277                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 278                                                    prot);
 279                 } else if (new_iomap == NULL) {
 280                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 281                                                     PAGE_KERNEL);
 282                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 283                                                    prot);
 284                 } else
 285                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 286                 if (ret)
 287                         goto out1;
 288         }
 289         mb();
 290 out2:
 291         ttm_bo_free_old_node(bo);
 292
 293         *old_mem = *new_mem;
 294         new_mem->mm_node = NULL;
 295
 296         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 297                 ttm_tt_unbind(ttm);
 298                 ttm_tt_destroy(ttm);
 299                 bo->ttm = NULL;
 300         }
 301
 302 out1:
 303         ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
 304 out:
 305         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 306         return ret;
 307 }
 308 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 309
 310 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 311 {
 312         kfree(bo);
 313 }
 314
 315 /**
 316  * ttm_buffer_object_transfer
 317  *
 318  * @bo: A pointer to a struct ttm_buffer_object.
 319  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 320  * holding the data of @bo with the old placement.
 321  *
 322  * This is a utility function that may be called after an accelerated move
 323  * has been scheduled. A new buffer object is created as a placeholder for
 324  * the old data while it's being copied. When that buffer object is idle,
 325  * it can be destroyed, releasing the space of the old placement.
 326  * Returns:
 327  * !0: Failure.
 328  */
 329
 330 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 331                                       struct ttm_buffer_object **new_obj)
 332 {
 333         struct ttm_buffer_object *fbo;
 334         struct ttm_bo_device *bdev = bo->bdev;
 335         struct ttm_bo_driver *driver = bdev->driver;
 336
 337         fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
 338         if (!fbo)
 339                 return -ENOMEM;
 340
 341         *fbo = *bo;
 342
 343         /**
 344          * Fix up members that we shouldn't copy directly:
 345          * TODO: Explicit member copy would probably be better here.
 346          */
 347
 348         spin_lock_init(&fbo->lock);
 349         init_waitqueue_head(&fbo->event_queue);
 350         INIT_LIST_HEAD(&fbo->ddestroy);
 351         INIT_LIST_HEAD(&fbo->lru);
 352         INIT_LIST_HEAD(&fbo->swap);
 353         fbo->vm_node = NULL;
 354
 355         fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 356         kref_init(&fbo->list_kref);
 357         kref_init(&fbo->kref);
 358         fbo->destroy = &ttm_transfered_destroy;
 359
 360         *new_obj = fbo;
 361         return 0;
 362 }
 363
 364 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 365 {
 366 #if defined(__i386__) || defined(__x86_64__)
 367         if (caching_flags & TTM_PL_FLAG_WC)
 368                 tmp = pgprot_writecombine(tmp);
 369         else if (boot_cpu_data.x86 > 3)
 370                 tmp = pgprot_noncached(tmp);
 371
 372 #elif defined(__powerpc__)
 373         if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
 374                 pgprot_val(tmp) |= _PAGE_NO_CACHE;
 375                 if (caching_flags & TTM_PL_FLAG_UNCACHED)
 376                         pgprot_val(tmp) |= _PAGE_GUARDED;
 377         }
 378 #endif
 379 #if defined(__ia64__)
 380         if (caching_flags & TTM_PL_FLAG_WC)
 381                 tmp = pgprot_writecombine(tmp);
 382         else
 383                 tmp = pgprot_noncached(tmp);
 384 #endif
 385 #if defined(__sparc__)
 386         if (!(caching_flags & TTM_PL_FLAG_CACHED))
 387                 tmp = pgprot_noncached(tmp);
 388 #endif
 389         return tmp;
 390 }
 391 EXPORT_SYMBOL(ttm_io_prot);
 392
 393 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 394                           unsigned long offset,
 395                           unsigned long size,
 396                           struct ttm_bo_kmap_obj *map)
 397 {
 398         struct ttm_mem_reg *mem = &bo->mem;
 399
 400         if (bo->mem.bus.addr) {
 401                 map->bo_kmap_type = ttm_bo_map_premapped;
 402                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 403         } else {
 404                 map->bo_kmap_type = ttm_bo_map_iomap;
 405                 if (mem->placement & TTM_PL_FLAG_WC)
 406                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 407                                                   size);
 408                 else
 409                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 410                                                        size);
 411         }
 412         return (!map->virtual) ? -ENOMEM : 0;
 413 }
 414
 415 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 416                            unsigned long start_page,
 417                            unsigned long num_pages,
 418                            struct ttm_bo_kmap_obj *map)
 419 {
 420         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
 421         struct ttm_tt *ttm = bo->ttm;
 422         struct page *d;
 423         int i;
 424
 425         BUG_ON(!ttm);
 426         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 427                 /*
 428                  * We're mapping a single page, and the desired
 429                  * page protection is consistent with the bo.
 430                  */
 431
 432                 map->bo_kmap_type = ttm_bo_map_kmap;
 433                 map->page = ttm_tt_get_page(ttm, start_page);
 434                 map->virtual = kmap(map->page);
 435         } else {
 436             /*
 437              * Populate the part we're mapping;
 438              */
 439                 for (i = start_page; i < start_page + num_pages; ++i) {
 440                         d = ttm_tt_get_page(ttm, i);
 441                         if (!d)
 442                                 return -ENOMEM;
 443                 }
 444
 445                 /*
 446                  * We need to use vmap to get the desired page protection
 447                  * or to make the buffer object look contiguous.
 448                  */
 449                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 450                         PAGE_KERNEL :
 451                         ttm_io_prot(mem->placement, PAGE_KERNEL);
 452                 map->bo_kmap_type = ttm_bo_map_vmap;
 453                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 454                                     0, prot);
 455         }
 456         return (!map->virtual) ? -ENOMEM : 0;
 457 }
 458
 459 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 460                 unsigned long start_page, unsigned long num_pages,
 461                 struct ttm_bo_kmap_obj *map)
 462 {
 463         unsigned long offset, size;
 464         int ret;
 465
 466         BUG_ON(!list_empty(&bo->swap));
 467         map->virtual = NULL;
 468         map->bo = bo;
 469         if (num_pages > bo->num_pages)
 470                 return -EINVAL;
 471         if (start_page > bo->num_pages)
 472                 return -EINVAL;
 473 #if 0
 474         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 475                 return -EPERM;
 476 #endif
 477         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 478         if (ret)
 479                 return ret;
 480         if (!bo->mem.bus.is_iomem) {
 481                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 482         } else {
 483                 offset = start_page << PAGE_SHIFT;
 484                 size = num_pages << PAGE_SHIFT;
 485                 return ttm_bo_ioremap(bo, offset, size, map);
 486         }
 487 }
 488 EXPORT_SYMBOL(ttm_bo_kmap);
 489
 490 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 491 {
 492         if (!map->virtual)
 493                 return;
 494         switch (map->bo_kmap_type) {
 495         case ttm_bo_map_iomap:
 496                 iounmap(map->virtual);
 497                 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 498                 break;
 499         case ttm_bo_map_vmap:
 500                 vunmap(map->virtual);
 501                 break;
 502         case ttm_bo_map_kmap:
 503                 kunmap(map->page);
 504                 break;
 505         case ttm_bo_map_premapped:
 506                 break;
 507         default:
 508                 BUG();
 509         }
 510         map->virtual = NULL;
 511         map->page = NULL;
 512 }
 513 EXPORT_SYMBOL(ttm_bo_kunmap);
 514
 515 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 516                               void *sync_obj,
 517                               void *sync_obj_arg,
 518                               bool evict, bool no_wait_reserve,
 519                               bool no_wait_gpu,
 520                               struct ttm_mem_reg *new_mem)
 521 {
 522         struct ttm_bo_device *bdev = bo->bdev;
 523         struct ttm_bo_driver *driver = bdev->driver;
 524         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 525         struct ttm_mem_reg *old_mem = &bo->mem;
 526         int ret;
 527         struct ttm_buffer_object *ghost_obj;
 528         void *tmp_obj = NULL;
 529
 530         spin_lock(&bo->lock);
 531         if (bo->sync_obj) {
 532                 tmp_obj = bo->sync_obj;
 533                 bo->sync_obj = NULL;
 534         }
 535         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 536         bo->sync_obj_arg = sync_obj_arg;
 537         if (evict) {
 538                 ret = ttm_bo_wait(bo, false, false, false);
 539                 spin_unlock(&bo->lock);
 540                 if (tmp_obj)
 541                         driver->sync_obj_unref(&tmp_obj);
 542                 if (ret)
 543                         return ret;
 544
 545                 ttm_bo_free_old_node(bo);
 546                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 547                     (bo->ttm != NULL)) {
 548                         ttm_tt_unbind(bo->ttm);
 549                         ttm_tt_destroy(bo->ttm);
 550                         bo->ttm = NULL;
 551                 }
 552         } else {
 553                 /**
 554                  * This should help pipeline ordinary buffer moves.
 555                  *
 556                  * Hang old buffer memory on a new buffer object,
 557                  * and leave it to be released when the GPU
 558                  * operation has completed.
 559                  */
 560
 561                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 562                 spin_unlock(&bo->lock);
 563                 if (tmp_obj)
 564                         driver->sync_obj_unref(&tmp_obj);
 565
 566                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 567                 if (ret)
 568                         return ret;
 569
 570                 /**
 571                  * If we're not moving to fixed memory, the TTM object
 572                  * needs to stay alive. Otherwhise hang it on the ghost
 573                  * bo to be unbound and destroyed.
 574                  */
 575
 576                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 577                         ghost_obj->ttm = NULL;
 578                 else
 579                         bo->ttm = NULL;
 580
 581                 ttm_bo_unreserve(ghost_obj);
 582                 ttm_bo_unref(&ghost_obj);
 583         }
 584
 585         *old_mem = *new_mem;
 586         new_mem->mm_node = NULL;
 587
 588         return 0;
 589 }
 590 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);