2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
32 #include <ttm/ttm_bo_api.h>
33 #include <ttm/ttm_bo_driver.h>
34 #include <ttm/ttm_placement.h>
35 #include <ttm/ttm_module.h>
36 #include <ttm/ttm_page_alloc.h>
38 #include <drm/radeon_drm.h>
39 #include <linux/seq_file.h>
40 #include <linux/slab.h>
41 #include <linux/swiotlb.h>
42 #include <linux/swap.h>
43 #include <linux/pagemap.h>
44 #include <linux/debugfs.h>
45 #include "radeon_reg.h"
48 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
50 static int radeon_ttm_debugfs_init(struct radeon_device
*rdev
);
51 static void radeon_ttm_debugfs_fini(struct radeon_device
*rdev
);
53 static struct radeon_device
*radeon_get_rdev(struct ttm_bo_device
*bdev
)
55 struct radeon_mman
*mman
;
56 struct radeon_device
*rdev
;
58 mman
= container_of(bdev
, struct radeon_mman
, bdev
);
59 rdev
= container_of(mman
, struct radeon_device
, mman
);
67 static int radeon_ttm_mem_global_init(struct drm_global_reference
*ref
)
69 return ttm_mem_global_init(ref
->object
);
72 static void radeon_ttm_mem_global_release(struct drm_global_reference
*ref
)
74 ttm_mem_global_release(ref
->object
);
77 static int radeon_ttm_global_init(struct radeon_device
*rdev
)
79 struct drm_global_reference
*global_ref
;
82 rdev
->mman
.mem_global_referenced
= false;
83 global_ref
= &rdev
->mman
.mem_global_ref
;
84 global_ref
->global_type
= DRM_GLOBAL_TTM_MEM
;
85 global_ref
->size
= sizeof(struct ttm_mem_global
);
86 global_ref
->init
= &radeon_ttm_mem_global_init
;
87 global_ref
->release
= &radeon_ttm_mem_global_release
;
88 r
= drm_global_item_ref(global_ref
);
90 DRM_ERROR("Failed setting up TTM memory accounting "
95 rdev
->mman
.bo_global_ref
.mem_glob
=
96 rdev
->mman
.mem_global_ref
.object
;
97 global_ref
= &rdev
->mman
.bo_global_ref
.ref
;
98 global_ref
->global_type
= DRM_GLOBAL_TTM_BO
;
99 global_ref
->size
= sizeof(struct ttm_bo_global
);
100 global_ref
->init
= &ttm_bo_global_init
;
101 global_ref
->release
= &ttm_bo_global_release
;
102 r
= drm_global_item_ref(global_ref
);
104 DRM_ERROR("Failed setting up TTM BO subsystem.\n");
105 drm_global_item_unref(&rdev
->mman
.mem_global_ref
);
109 rdev
->mman
.mem_global_referenced
= true;
113 static void radeon_ttm_global_fini(struct radeon_device
*rdev
)
115 if (rdev
->mman
.mem_global_referenced
) {
116 drm_global_item_unref(&rdev
->mman
.bo_global_ref
.ref
);
117 drm_global_item_unref(&rdev
->mman
.mem_global_ref
);
118 rdev
->mman
.mem_global_referenced
= false;
122 static int radeon_invalidate_caches(struct ttm_bo_device
*bdev
, uint32_t flags
)
127 static int radeon_init_mem_type(struct ttm_bo_device
*bdev
, uint32_t type
,
128 struct ttm_mem_type_manager
*man
)
130 struct radeon_device
*rdev
;
132 rdev
= radeon_get_rdev(bdev
);
137 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
138 man
->available_caching
= TTM_PL_MASK_CACHING
;
139 man
->default_caching
= TTM_PL_FLAG_CACHED
;
142 man
->func
= &ttm_bo_manager_func
;
143 man
->gpu_offset
= rdev
->mc
.gtt_start
;
144 man
->available_caching
= TTM_PL_MASK_CACHING
;
145 man
->default_caching
= TTM_PL_FLAG_CACHED
;
146 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
| TTM_MEMTYPE_FLAG_CMA
;
147 #if IS_ENABLED(CONFIG_AGP)
148 if (rdev
->flags
& RADEON_IS_AGP
) {
149 if (!rdev
->ddev
->agp
) {
150 DRM_ERROR("AGP is not enabled for memory type %u\n",
154 if (!rdev
->ddev
->agp
->cant_use_aperture
)
155 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
156 man
->available_caching
= TTM_PL_FLAG_UNCACHED
|
158 man
->default_caching
= TTM_PL_FLAG_WC
;
163 /* "On-card" video ram */
164 man
->func
= &ttm_bo_manager_func
;
165 man
->gpu_offset
= rdev
->mc
.vram_start
;
166 man
->flags
= TTM_MEMTYPE_FLAG_FIXED
|
167 TTM_MEMTYPE_FLAG_MAPPABLE
;
168 man
->available_caching
= TTM_PL_FLAG_UNCACHED
| TTM_PL_FLAG_WC
;
169 man
->default_caching
= TTM_PL_FLAG_WC
;
172 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type
);
178 static void radeon_evict_flags(struct ttm_buffer_object
*bo
,
179 struct ttm_placement
*placement
)
181 static struct ttm_place placements
= {
184 .flags
= TTM_PL_MASK_CACHING
| TTM_PL_FLAG_SYSTEM
187 struct radeon_bo
*rbo
;
189 if (!radeon_ttm_bo_is_radeon_bo(bo
)) {
190 placement
->placement
= &placements
;
191 placement
->busy_placement
= &placements
;
192 placement
->num_placement
= 1;
193 placement
->num_busy_placement
= 1;
196 rbo
= container_of(bo
, struct radeon_bo
, tbo
);
197 switch (bo
->mem
.mem_type
) {
199 if (rbo
->rdev
->ring
[radeon_copy_ring_index(rbo
->rdev
)].ready
== false)
200 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_CPU
);
201 else if (rbo
->rdev
->mc
.visible_vram_size
< rbo
->rdev
->mc
.real_vram_size
&&
202 bo
->mem
.start
< (rbo
->rdev
->mc
.visible_vram_size
>> PAGE_SHIFT
)) {
203 unsigned fpfn
= rbo
->rdev
->mc
.visible_vram_size
>> PAGE_SHIFT
;
206 /* Try evicting to the CPU inaccessible part of VRAM
207 * first, but only set GTT as busy placement, so this
208 * BO will be evicted to GTT rather than causing other
209 * BOs to be evicted from VRAM
211 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_VRAM
|
212 RADEON_GEM_DOMAIN_GTT
);
213 rbo
->placement
.num_busy_placement
= 0;
214 for (i
= 0; i
< rbo
->placement
.num_placement
; i
++) {
215 if (rbo
->placements
[i
].flags
& TTM_PL_FLAG_VRAM
) {
216 if (rbo
->placements
[0].fpfn
< fpfn
)
217 rbo
->placements
[0].fpfn
= fpfn
;
219 rbo
->placement
.busy_placement
=
221 rbo
->placement
.num_busy_placement
= 1;
225 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_GTT
);
229 radeon_ttm_placement_from_domain(rbo
, RADEON_GEM_DOMAIN_CPU
);
231 *placement
= rbo
->placement
;
234 static int radeon_verify_access(struct ttm_buffer_object
*bo
, struct file
*filp
)
236 struct radeon_bo
*rbo
= container_of(bo
, struct radeon_bo
, tbo
);
238 return drm_vma_node_verify_access(&rbo
->gem_base
.vma_node
, filp
);
241 static void radeon_move_null(struct ttm_buffer_object
*bo
,
242 struct ttm_mem_reg
*new_mem
)
244 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
246 BUG_ON(old_mem
->mm_node
!= NULL
);
248 new_mem
->mm_node
= NULL
;
251 static int radeon_move_blit(struct ttm_buffer_object
*bo
,
252 bool evict
, bool no_wait_gpu
,
253 struct ttm_mem_reg
*new_mem
,
254 struct ttm_mem_reg
*old_mem
)
256 struct radeon_device
*rdev
;
257 uint64_t old_start
, new_start
;
258 struct radeon_fence
*fence
;
262 rdev
= radeon_get_rdev(bo
->bdev
);
263 ridx
= radeon_copy_ring_index(rdev
);
264 old_start
= old_mem
->start
<< PAGE_SHIFT
;
265 new_start
= new_mem
->start
<< PAGE_SHIFT
;
267 switch (old_mem
->mem_type
) {
269 old_start
+= rdev
->mc
.vram_start
;
272 old_start
+= rdev
->mc
.gtt_start
;
275 DRM_ERROR("Unknown placement %d\n", old_mem
->mem_type
);
278 switch (new_mem
->mem_type
) {
280 new_start
+= rdev
->mc
.vram_start
;
283 new_start
+= rdev
->mc
.gtt_start
;
286 DRM_ERROR("Unknown placement %d\n", old_mem
->mem_type
);
289 if (!rdev
->ring
[ridx
].ready
) {
290 DRM_ERROR("Trying to move memory with ring turned off.\n");
294 BUILD_BUG_ON((PAGE_SIZE
% RADEON_GPU_PAGE_SIZE
) != 0);
296 num_pages
= new_mem
->num_pages
* (PAGE_SIZE
/ RADEON_GPU_PAGE_SIZE
);
297 fence
= radeon_copy(rdev
, old_start
, new_start
, num_pages
, bo
->resv
);
299 return PTR_ERR(fence
);
301 r
= ttm_bo_move_accel_cleanup(bo
, &fence
->base
,
302 evict
, no_wait_gpu
, new_mem
);
303 radeon_fence_unref(&fence
);
307 static int radeon_move_vram_ram(struct ttm_buffer_object
*bo
,
308 bool evict
, bool interruptible
,
310 struct ttm_mem_reg
*new_mem
)
312 struct radeon_device
*rdev
;
313 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
314 struct ttm_mem_reg tmp_mem
;
315 struct ttm_place placements
;
316 struct ttm_placement placement
;
319 rdev
= radeon_get_rdev(bo
->bdev
);
321 tmp_mem
.mm_node
= NULL
;
322 placement
.num_placement
= 1;
323 placement
.placement
= &placements
;
324 placement
.num_busy_placement
= 1;
325 placement
.busy_placement
= &placements
;
328 placements
.flags
= TTM_PL_MASK_CACHING
| TTM_PL_FLAG_TT
;
329 r
= ttm_bo_mem_space(bo
, &placement
, &tmp_mem
,
330 interruptible
, no_wait_gpu
);
335 r
= ttm_tt_set_placement_caching(bo
->ttm
, tmp_mem
.placement
);
340 r
= ttm_tt_bind(bo
->ttm
, &tmp_mem
);
344 r
= radeon_move_blit(bo
, true, no_wait_gpu
, &tmp_mem
, old_mem
);
348 r
= ttm_bo_move_ttm(bo
, true, no_wait_gpu
, new_mem
);
350 ttm_bo_mem_put(bo
, &tmp_mem
);
354 static int radeon_move_ram_vram(struct ttm_buffer_object
*bo
,
355 bool evict
, bool interruptible
,
357 struct ttm_mem_reg
*new_mem
)
359 struct radeon_device
*rdev
;
360 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
361 struct ttm_mem_reg tmp_mem
;
362 struct ttm_placement placement
;
363 struct ttm_place placements
;
366 rdev
= radeon_get_rdev(bo
->bdev
);
368 tmp_mem
.mm_node
= NULL
;
369 placement
.num_placement
= 1;
370 placement
.placement
= &placements
;
371 placement
.num_busy_placement
= 1;
372 placement
.busy_placement
= &placements
;
375 placements
.flags
= TTM_PL_MASK_CACHING
| TTM_PL_FLAG_TT
;
376 r
= ttm_bo_mem_space(bo
, &placement
, &tmp_mem
,
377 interruptible
, no_wait_gpu
);
381 r
= ttm_bo_move_ttm(bo
, true, no_wait_gpu
, &tmp_mem
);
385 r
= radeon_move_blit(bo
, true, no_wait_gpu
, new_mem
, old_mem
);
390 ttm_bo_mem_put(bo
, &tmp_mem
);
394 static int radeon_bo_move(struct ttm_buffer_object
*bo
,
395 bool evict
, bool interruptible
,
397 struct ttm_mem_reg
*new_mem
)
399 struct radeon_device
*rdev
;
400 struct radeon_bo
*rbo
;
401 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
404 /* Can't move a pinned BO */
405 rbo
= container_of(bo
, struct radeon_bo
, tbo
);
406 if (WARN_ON_ONCE(rbo
->pin_count
> 0))
409 rdev
= radeon_get_rdev(bo
->bdev
);
410 if (old_mem
->mem_type
== TTM_PL_SYSTEM
&& bo
->ttm
== NULL
) {
411 radeon_move_null(bo
, new_mem
);
414 if ((old_mem
->mem_type
== TTM_PL_TT
&&
415 new_mem
->mem_type
== TTM_PL_SYSTEM
) ||
416 (old_mem
->mem_type
== TTM_PL_SYSTEM
&&
417 new_mem
->mem_type
== TTM_PL_TT
)) {
419 radeon_move_null(bo
, new_mem
);
422 if (!rdev
->ring
[radeon_copy_ring_index(rdev
)].ready
||
423 rdev
->asic
->copy
.copy
== NULL
) {
428 if (old_mem
->mem_type
== TTM_PL_VRAM
&&
429 new_mem
->mem_type
== TTM_PL_SYSTEM
) {
430 r
= radeon_move_vram_ram(bo
, evict
, interruptible
,
431 no_wait_gpu
, new_mem
);
432 } else if (old_mem
->mem_type
== TTM_PL_SYSTEM
&&
433 new_mem
->mem_type
== TTM_PL_VRAM
) {
434 r
= radeon_move_ram_vram(bo
, evict
, interruptible
,
435 no_wait_gpu
, new_mem
);
437 r
= radeon_move_blit(bo
, evict
, no_wait_gpu
, new_mem
, old_mem
);
442 r
= ttm_bo_move_memcpy(bo
, evict
, no_wait_gpu
, new_mem
);
448 /* update statistics */
449 atomic64_add((u64
)bo
->num_pages
<< PAGE_SHIFT
, &rdev
->num_bytes_moved
);
453 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
455 struct ttm_mem_type_manager
*man
= &bdev
->man
[mem
->mem_type
];
456 struct radeon_device
*rdev
= radeon_get_rdev(bdev
);
458 mem
->bus
.addr
= NULL
;
460 mem
->bus
.size
= mem
->num_pages
<< PAGE_SHIFT
;
462 mem
->bus
.is_iomem
= false;
463 if (!(man
->flags
& TTM_MEMTYPE_FLAG_MAPPABLE
))
465 switch (mem
->mem_type
) {
470 #if IS_ENABLED(CONFIG_AGP)
471 if (rdev
->flags
& RADEON_IS_AGP
) {
472 /* RADEON_IS_AGP is set only if AGP is active */
473 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
474 mem
->bus
.base
= rdev
->mc
.agp_base
;
475 mem
->bus
.is_iomem
= !rdev
->ddev
->agp
->cant_use_aperture
;
480 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
481 /* check if it's visible */
482 if ((mem
->bus
.offset
+ mem
->bus
.size
) > rdev
->mc
.visible_vram_size
)
484 mem
->bus
.base
= rdev
->mc
.aper_base
;
485 mem
->bus
.is_iomem
= true;
488 * Alpha: use bus.addr to hold the ioremap() return,
489 * so we can modify bus.base below.
491 if (mem
->placement
& TTM_PL_FLAG_WC
)
493 ioremap_wc(mem
->bus
.base
+ mem
->bus
.offset
,
497 ioremap_nocache(mem
->bus
.base
+ mem
->bus
.offset
,
501 * Alpha: Use just the bus offset plus
502 * the hose/domain memory base for bus.base.
503 * It then can be used to build PTEs for VRAM
504 * access, as done in ttm_bo_vm_fault().
506 mem
->bus
.base
= (mem
->bus
.base
& 0x0ffffffffUL
) +
507 rdev
->ddev
->hose
->dense_mem_base
;
516 static void radeon_ttm_io_mem_free(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
521 * TTM backend functions.
523 struct radeon_ttm_tt
{
524 struct ttm_dma_tt ttm
;
525 struct radeon_device
*rdev
;
529 struct mm_struct
*usermm
;
533 /* prepare the sg table with the user pages */
534 static int radeon_ttm_tt_pin_userptr(struct ttm_tt
*ttm
)
536 struct radeon_device
*rdev
= radeon_get_rdev(ttm
->bdev
);
537 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
538 unsigned pinned
= 0, nents
;
541 int write
= !(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
);
542 enum dma_data_direction direction
= write
?
543 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
;
545 if (current
->mm
!= gtt
->usermm
)
548 if (gtt
->userflags
& RADEON_GEM_USERPTR_ANONONLY
) {
549 /* check that we only pin down anonymous memory
550 to prevent problems with writeback */
551 unsigned long end
= gtt
->userptr
+ ttm
->num_pages
* PAGE_SIZE
;
552 struct vm_area_struct
*vma
;
553 vma
= find_vma(gtt
->usermm
, gtt
->userptr
);
554 if (!vma
|| vma
->vm_file
|| vma
->vm_end
< end
)
559 unsigned num_pages
= ttm
->num_pages
- pinned
;
560 uint64_t userptr
= gtt
->userptr
+ pinned
* PAGE_SIZE
;
561 struct page
**pages
= ttm
->pages
+ pinned
;
563 r
= get_user_pages(userptr
, num_pages
, write
, 0, pages
, NULL
);
569 } while (pinned
< ttm
->num_pages
);
571 r
= sg_alloc_table_from_pages(ttm
->sg
, ttm
->pages
, ttm
->num_pages
, 0,
572 ttm
->num_pages
<< PAGE_SHIFT
,
578 nents
= dma_map_sg(rdev
->dev
, ttm
->sg
->sgl
, ttm
->sg
->nents
, direction
);
579 if (nents
!= ttm
->sg
->nents
)
582 drm_prime_sg_to_page_addr_arrays(ttm
->sg
, ttm
->pages
,
583 gtt
->ttm
.dma_address
, ttm
->num_pages
);
591 release_pages(ttm
->pages
, pinned
, 0);
595 static void radeon_ttm_tt_unpin_userptr(struct ttm_tt
*ttm
)
597 struct radeon_device
*rdev
= radeon_get_rdev(ttm
->bdev
);
598 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
599 struct sg_page_iter sg_iter
;
601 int write
= !(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
);
602 enum dma_data_direction direction
= write
?
603 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
;
605 /* double check that we don't free the table twice */
609 /* free the sg table and pages again */
610 dma_unmap_sg(rdev
->dev
, ttm
->sg
->sgl
, ttm
->sg
->nents
, direction
);
612 for_each_sg_page(ttm
->sg
->sgl
, &sg_iter
, ttm
->sg
->nents
, 0) {
613 struct page
*page
= sg_page_iter_page(&sg_iter
);
614 if (!(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
))
615 set_page_dirty(page
);
617 mark_page_accessed(page
);
618 page_cache_release(page
);
621 sg_free_table(ttm
->sg
);
624 static int radeon_ttm_backend_bind(struct ttm_tt
*ttm
,
625 struct ttm_mem_reg
*bo_mem
)
627 struct radeon_ttm_tt
*gtt
= (void*)ttm
;
628 uint32_t flags
= RADEON_GART_PAGE_VALID
| RADEON_GART_PAGE_READ
|
629 RADEON_GART_PAGE_WRITE
;
633 radeon_ttm_tt_pin_userptr(ttm
);
634 flags
&= ~RADEON_GART_PAGE_WRITE
;
637 gtt
->offset
= (unsigned long)(bo_mem
->start
<< PAGE_SHIFT
);
638 if (!ttm
->num_pages
) {
639 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
640 ttm
->num_pages
, bo_mem
, ttm
);
642 if (ttm
->caching_state
== tt_cached
)
643 flags
|= RADEON_GART_PAGE_SNOOP
;
644 r
= radeon_gart_bind(gtt
->rdev
, gtt
->offset
, ttm
->num_pages
,
645 ttm
->pages
, gtt
->ttm
.dma_address
, flags
);
647 DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
648 ttm
->num_pages
, (unsigned)gtt
->offset
);
654 static int radeon_ttm_backend_unbind(struct ttm_tt
*ttm
)
656 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
658 radeon_gart_unbind(gtt
->rdev
, gtt
->offset
, ttm
->num_pages
);
661 radeon_ttm_tt_unpin_userptr(ttm
);
666 static void radeon_ttm_backend_destroy(struct ttm_tt
*ttm
)
668 struct radeon_ttm_tt
*gtt
= (void *)ttm
;
670 ttm_dma_tt_fini(>t
->ttm
);
674 static struct ttm_backend_func radeon_backend_func
= {
675 .bind
= &radeon_ttm_backend_bind
,
676 .unbind
= &radeon_ttm_backend_unbind
,
677 .destroy
= &radeon_ttm_backend_destroy
,
680 static struct ttm_tt
*radeon_ttm_tt_create(struct ttm_bo_device
*bdev
,
681 unsigned long size
, uint32_t page_flags
,
682 struct page
*dummy_read_page
)
684 struct radeon_device
*rdev
;
685 struct radeon_ttm_tt
*gtt
;
687 rdev
= radeon_get_rdev(bdev
);
688 #if IS_ENABLED(CONFIG_AGP)
689 if (rdev
->flags
& RADEON_IS_AGP
) {
690 return ttm_agp_tt_create(bdev
, rdev
->ddev
->agp
->bridge
,
691 size
, page_flags
, dummy_read_page
);
695 gtt
= kzalloc(sizeof(struct radeon_ttm_tt
), GFP_KERNEL
);
699 gtt
->ttm
.ttm
.func
= &radeon_backend_func
;
701 if (ttm_dma_tt_init(>t
->ttm
, bdev
, size
, page_flags
, dummy_read_page
)) {
705 return >t
->ttm
.ttm
;
708 static struct radeon_ttm_tt
*radeon_ttm_tt_to_gtt(struct ttm_tt
*ttm
)
710 if (!ttm
|| ttm
->func
!= &radeon_backend_func
)
712 return (struct radeon_ttm_tt
*)ttm
;
715 static int radeon_ttm_tt_populate(struct ttm_tt
*ttm
)
717 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
718 struct radeon_device
*rdev
;
721 bool slave
= !!(ttm
->page_flags
& TTM_PAGE_FLAG_SG
);
723 if (ttm
->state
!= tt_unpopulated
)
726 if (gtt
&& gtt
->userptr
) {
727 ttm
->sg
= kzalloc(sizeof(struct sg_table
), GFP_KERNEL
);
731 ttm
->page_flags
|= TTM_PAGE_FLAG_SG
;
732 ttm
->state
= tt_unbound
;
736 if (slave
&& ttm
->sg
) {
737 drm_prime_sg_to_page_addr_arrays(ttm
->sg
, ttm
->pages
,
738 gtt
->ttm
.dma_address
, ttm
->num_pages
);
739 ttm
->state
= tt_unbound
;
743 rdev
= radeon_get_rdev(ttm
->bdev
);
744 #if IS_ENABLED(CONFIG_AGP)
745 if (rdev
->flags
& RADEON_IS_AGP
) {
746 return ttm_agp_tt_populate(ttm
);
750 #ifdef CONFIG_SWIOTLB
751 if (swiotlb_nr_tbl()) {
752 return ttm_dma_populate(>t
->ttm
, rdev
->dev
);
756 r
= ttm_pool_populate(ttm
);
761 for (i
= 0; i
< ttm
->num_pages
; i
++) {
762 gtt
->ttm
.dma_address
[i
] = pci_map_page(rdev
->pdev
, ttm
->pages
[i
],
764 PCI_DMA_BIDIRECTIONAL
);
765 if (pci_dma_mapping_error(rdev
->pdev
, gtt
->ttm
.dma_address
[i
])) {
767 pci_unmap_page(rdev
->pdev
, gtt
->ttm
.dma_address
[i
],
768 PAGE_SIZE
, PCI_DMA_BIDIRECTIONAL
);
769 gtt
->ttm
.dma_address
[i
] = 0;
771 ttm_pool_unpopulate(ttm
);
778 static void radeon_ttm_tt_unpopulate(struct ttm_tt
*ttm
)
780 struct radeon_device
*rdev
;
781 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
783 bool slave
= !!(ttm
->page_flags
& TTM_PAGE_FLAG_SG
);
785 if (gtt
&& gtt
->userptr
) {
787 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SG
;
794 rdev
= radeon_get_rdev(ttm
->bdev
);
795 #if IS_ENABLED(CONFIG_AGP)
796 if (rdev
->flags
& RADEON_IS_AGP
) {
797 ttm_agp_tt_unpopulate(ttm
);
802 #ifdef CONFIG_SWIOTLB
803 if (swiotlb_nr_tbl()) {
804 ttm_dma_unpopulate(>t
->ttm
, rdev
->dev
);
809 for (i
= 0; i
< ttm
->num_pages
; i
++) {
810 if (gtt
->ttm
.dma_address
[i
]) {
811 pci_unmap_page(rdev
->pdev
, gtt
->ttm
.dma_address
[i
],
812 PAGE_SIZE
, PCI_DMA_BIDIRECTIONAL
);
816 ttm_pool_unpopulate(ttm
);
819 int radeon_ttm_tt_set_userptr(struct ttm_tt
*ttm
, uint64_t addr
,
822 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
828 gtt
->usermm
= current
->mm
;
829 gtt
->userflags
= flags
;
833 bool radeon_ttm_tt_has_userptr(struct ttm_tt
*ttm
)
835 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
840 return !!gtt
->userptr
;
843 bool radeon_ttm_tt_is_readonly(struct ttm_tt
*ttm
)
845 struct radeon_ttm_tt
*gtt
= radeon_ttm_tt_to_gtt(ttm
);
850 return !!(gtt
->userflags
& RADEON_GEM_USERPTR_READONLY
);
853 static struct ttm_bo_driver radeon_bo_driver
= {
854 .ttm_tt_create
= &radeon_ttm_tt_create
,
855 .ttm_tt_populate
= &radeon_ttm_tt_populate
,
856 .ttm_tt_unpopulate
= &radeon_ttm_tt_unpopulate
,
857 .invalidate_caches
= &radeon_invalidate_caches
,
858 .init_mem_type
= &radeon_init_mem_type
,
859 .evict_flags
= &radeon_evict_flags
,
860 .move
= &radeon_bo_move
,
861 .verify_access
= &radeon_verify_access
,
862 .move_notify
= &radeon_bo_move_notify
,
863 .fault_reserve_notify
= &radeon_bo_fault_reserve_notify
,
864 .io_mem_reserve
= &radeon_ttm_io_mem_reserve
,
865 .io_mem_free
= &radeon_ttm_io_mem_free
,
868 int radeon_ttm_init(struct radeon_device
*rdev
)
872 r
= radeon_ttm_global_init(rdev
);
876 /* No others user of address space so set it to 0 */
877 r
= ttm_bo_device_init(&rdev
->mman
.bdev
,
878 rdev
->mman
.bo_global_ref
.ref
.object
,
880 rdev
->ddev
->anon_inode
->i_mapping
,
881 DRM_FILE_PAGE_OFFSET
,
884 DRM_ERROR("failed initializing buffer object driver(%d).\n", r
);
887 rdev
->mman
.initialized
= true;
888 r
= ttm_bo_init_mm(&rdev
->mman
.bdev
, TTM_PL_VRAM
,
889 rdev
->mc
.real_vram_size
>> PAGE_SHIFT
);
891 DRM_ERROR("Failed initializing VRAM heap.\n");
894 /* Change the size here instead of the init above so only lpfn is affected */
895 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
897 r
= radeon_bo_create(rdev
, 256 * 1024, PAGE_SIZE
, true,
898 RADEON_GEM_DOMAIN_VRAM
, 0, NULL
,
899 NULL
, &rdev
->stollen_vga_memory
);
903 r
= radeon_bo_reserve(rdev
->stollen_vga_memory
, false);
906 r
= radeon_bo_pin(rdev
->stollen_vga_memory
, RADEON_GEM_DOMAIN_VRAM
, NULL
);
907 radeon_bo_unreserve(rdev
->stollen_vga_memory
);
909 radeon_bo_unref(&rdev
->stollen_vga_memory
);
912 DRM_INFO("radeon: %uM of VRAM memory ready\n",
913 (unsigned) (rdev
->mc
.real_vram_size
/ (1024 * 1024)));
914 r
= ttm_bo_init_mm(&rdev
->mman
.bdev
, TTM_PL_TT
,
915 rdev
->mc
.gtt_size
>> PAGE_SHIFT
);
917 DRM_ERROR("Failed initializing GTT heap.\n");
920 DRM_INFO("radeon: %uM of GTT memory ready.\n",
921 (unsigned)(rdev
->mc
.gtt_size
/ (1024 * 1024)));
923 r
= radeon_ttm_debugfs_init(rdev
);
925 DRM_ERROR("Failed to init debugfs\n");
931 void radeon_ttm_fini(struct radeon_device
*rdev
)
935 if (!rdev
->mman
.initialized
)
937 radeon_ttm_debugfs_fini(rdev
);
938 if (rdev
->stollen_vga_memory
) {
939 r
= radeon_bo_reserve(rdev
->stollen_vga_memory
, false);
941 radeon_bo_unpin(rdev
->stollen_vga_memory
);
942 radeon_bo_unreserve(rdev
->stollen_vga_memory
);
944 radeon_bo_unref(&rdev
->stollen_vga_memory
);
946 ttm_bo_clean_mm(&rdev
->mman
.bdev
, TTM_PL_VRAM
);
947 ttm_bo_clean_mm(&rdev
->mman
.bdev
, TTM_PL_TT
);
948 ttm_bo_device_release(&rdev
->mman
.bdev
);
949 radeon_gart_fini(rdev
);
950 radeon_ttm_global_fini(rdev
);
951 rdev
->mman
.initialized
= false;
952 DRM_INFO("radeon: ttm finalized\n");
955 /* this should only be called at bootup or when userspace
957 void radeon_ttm_set_active_vram_size(struct radeon_device
*rdev
, u64 size
)
959 struct ttm_mem_type_manager
*man
;
961 if (!rdev
->mman
.initialized
)
964 man
= &rdev
->mman
.bdev
.man
[TTM_PL_VRAM
];
965 /* this just adjusts TTM size idea, which sets lpfn to the correct value */
966 man
->size
= size
>> PAGE_SHIFT
;
969 static struct vm_operations_struct radeon_ttm_vm_ops
;
970 static const struct vm_operations_struct
*ttm_vm_ops
= NULL
;
972 static int radeon_ttm_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
974 struct ttm_buffer_object
*bo
;
975 struct radeon_device
*rdev
;
978 bo
= (struct ttm_buffer_object
*)vma
->vm_private_data
;
980 return VM_FAULT_NOPAGE
;
982 rdev
= radeon_get_rdev(bo
->bdev
);
983 down_read(&rdev
->pm
.mclk_lock
);
984 r
= ttm_vm_ops
->fault(vma
, vmf
);
985 up_read(&rdev
->pm
.mclk_lock
);
989 int radeon_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
991 struct drm_file
*file_priv
;
992 struct radeon_device
*rdev
;
995 if (unlikely(vma
->vm_pgoff
< DRM_FILE_PAGE_OFFSET
)) {
999 file_priv
= filp
->private_data
;
1000 rdev
= file_priv
->minor
->dev
->dev_private
;
1004 r
= ttm_bo_mmap(filp
, vma
, &rdev
->mman
.bdev
);
1005 if (unlikely(r
!= 0)) {
1008 if (unlikely(ttm_vm_ops
== NULL
)) {
1009 ttm_vm_ops
= vma
->vm_ops
;
1010 radeon_ttm_vm_ops
= *ttm_vm_ops
;
1011 radeon_ttm_vm_ops
.fault
= &radeon_ttm_fault
;
1013 vma
->vm_ops
= &radeon_ttm_vm_ops
;
1017 #if defined(CONFIG_DEBUG_FS)
1019 static int radeon_mm_dump_table(struct seq_file
*m
, void *data
)
1021 struct drm_info_node
*node
= (struct drm_info_node
*)m
->private;
1022 unsigned ttm_pl
= *(int *)node
->info_ent
->data
;
1023 struct drm_device
*dev
= node
->minor
->dev
;
1024 struct radeon_device
*rdev
= dev
->dev_private
;
1025 struct drm_mm
*mm
= (struct drm_mm
*)rdev
->mman
.bdev
.man
[ttm_pl
].priv
;
1027 struct ttm_bo_global
*glob
= rdev
->mman
.bdev
.glob
;
1029 spin_lock(&glob
->lru_lock
);
1030 ret
= drm_mm_dump_table(m
, mm
);
1031 spin_unlock(&glob
->lru_lock
);
1035 static int ttm_pl_vram
= TTM_PL_VRAM
;
1036 static int ttm_pl_tt
= TTM_PL_TT
;
1038 static struct drm_info_list radeon_ttm_debugfs_list
[] = {
1039 {"radeon_vram_mm", radeon_mm_dump_table
, 0, &ttm_pl_vram
},
1040 {"radeon_gtt_mm", radeon_mm_dump_table
, 0, &ttm_pl_tt
},
1041 {"ttm_page_pool", ttm_page_alloc_debugfs
, 0, NULL
},
1042 #ifdef CONFIG_SWIOTLB
1043 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs
, 0, NULL
}
1047 static int radeon_ttm_vram_open(struct inode
*inode
, struct file
*filep
)
1049 struct radeon_device
*rdev
= inode
->i_private
;
1050 i_size_write(inode
, rdev
->mc
.mc_vram_size
);
1051 filep
->private_data
= inode
->i_private
;
1055 static ssize_t
radeon_ttm_vram_read(struct file
*f
, char __user
*buf
,
1056 size_t size
, loff_t
*pos
)
1058 struct radeon_device
*rdev
= f
->private_data
;
1062 if (size
& 0x3 || *pos
& 0x3)
1066 unsigned long flags
;
1069 if (*pos
>= rdev
->mc
.mc_vram_size
)
1072 spin_lock_irqsave(&rdev
->mmio_idx_lock
, flags
);
1073 WREG32(RADEON_MM_INDEX
, ((uint32_t)*pos
) | 0x80000000);
1074 if (rdev
->family
>= CHIP_CEDAR
)
1075 WREG32(EVERGREEN_MM_INDEX_HI
, *pos
>> 31);
1076 value
= RREG32(RADEON_MM_DATA
);
1077 spin_unlock_irqrestore(&rdev
->mmio_idx_lock
, flags
);
1079 r
= put_user(value
, (uint32_t *)buf
);
1092 static const struct file_operations radeon_ttm_vram_fops
= {
1093 .owner
= THIS_MODULE
,
1094 .open
= radeon_ttm_vram_open
,
1095 .read
= radeon_ttm_vram_read
,
1096 .llseek
= default_llseek
1099 static int radeon_ttm_gtt_open(struct inode
*inode
, struct file
*filep
)
1101 struct radeon_device
*rdev
= inode
->i_private
;
1102 i_size_write(inode
, rdev
->mc
.gtt_size
);
1103 filep
->private_data
= inode
->i_private
;
1107 static ssize_t
radeon_ttm_gtt_read(struct file
*f
, char __user
*buf
,
1108 size_t size
, loff_t
*pos
)
1110 struct radeon_device
*rdev
= f
->private_data
;
1115 loff_t p
= *pos
/ PAGE_SIZE
;
1116 unsigned off
= *pos
& ~PAGE_MASK
;
1117 size_t cur_size
= min_t(size_t, size
, PAGE_SIZE
- off
);
1121 if (p
>= rdev
->gart
.num_cpu_pages
)
1124 page
= rdev
->gart
.pages
[p
];
1129 r
= copy_to_user(buf
, ptr
, cur_size
);
1130 kunmap(rdev
->gart
.pages
[p
]);
1132 r
= clear_user(buf
, cur_size
);
1146 static const struct file_operations radeon_ttm_gtt_fops
= {
1147 .owner
= THIS_MODULE
,
1148 .open
= radeon_ttm_gtt_open
,
1149 .read
= radeon_ttm_gtt_read
,
1150 .llseek
= default_llseek
1155 static int radeon_ttm_debugfs_init(struct radeon_device
*rdev
)
1157 #if defined(CONFIG_DEBUG_FS)
1160 struct drm_minor
*minor
= rdev
->ddev
->primary
;
1161 struct dentry
*ent
, *root
= minor
->debugfs_root
;
1163 ent
= debugfs_create_file("radeon_vram", S_IFREG
| S_IRUGO
, root
,
1164 rdev
, &radeon_ttm_vram_fops
);
1166 return PTR_ERR(ent
);
1167 rdev
->mman
.vram
= ent
;
1169 ent
= debugfs_create_file("radeon_gtt", S_IFREG
| S_IRUGO
, root
,
1170 rdev
, &radeon_ttm_gtt_fops
);
1172 return PTR_ERR(ent
);
1173 rdev
->mman
.gtt
= ent
;
1175 count
= ARRAY_SIZE(radeon_ttm_debugfs_list
);
1177 #ifdef CONFIG_SWIOTLB
1178 if (!swiotlb_nr_tbl())
1182 return radeon_debugfs_add_files(rdev
, radeon_ttm_debugfs_list
, count
);
1189 static void radeon_ttm_debugfs_fini(struct radeon_device
*rdev
)
1191 #if defined(CONFIG_DEBUG_FS)
1193 debugfs_remove(rdev
->mman
.vram
);
1194 rdev
->mman
.vram
= NULL
;
1196 debugfs_remove(rdev
->mman
.gtt
);
1197 rdev
->mman
.gtt
= NULL
;