2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
32 #include <linux/list.h>
34 #include "radeon_drm.h"
37 struct radeon_object
{
38 struct ttm_buffer_object tobj
;
39 struct list_head list
;
40 struct radeon_device
*rdev
;
41 struct drm_gem_object
*gobj
;
42 struct ttm_bo_kmap_obj kmap
;
47 uint32_t tiling_flags
;
52 int radeon_ttm_init(struct radeon_device
*rdev
);
53 void radeon_ttm_fini(struct radeon_device
*rdev
);
56 * To exclude mutual BO access we rely on bo_reserve exclusion, as all
57 * function are calling it.
60 static int radeon_object_reserve(struct radeon_object
*robj
, bool interruptible
)
62 return ttm_bo_reserve(&robj
->tobj
, interruptible
, false, false, 0);
65 static void radeon_object_unreserve(struct radeon_object
*robj
)
67 ttm_bo_unreserve(&robj
->tobj
);
70 static void radeon_ttm_object_object_destroy(struct ttm_buffer_object
*tobj
)
72 struct radeon_object
*robj
;
74 robj
= container_of(tobj
, struct radeon_object
, tobj
);
75 list_del_init(&robj
->list
);
76 radeon_object_clear_surface_reg(robj
);
80 static inline void radeon_object_gpu_addr(struct radeon_object
*robj
)
82 /* Default gpu address */
83 robj
->gpu_addr
= 0xFFFFFFFFFFFFFFFFULL
;
84 if (robj
->tobj
.mem
.mm_node
== NULL
) {
87 robj
->gpu_addr
= ((u64
)robj
->tobj
.mem
.mm_node
->start
) << PAGE_SHIFT
;
88 switch (robj
->tobj
.mem
.mem_type
) {
90 robj
->gpu_addr
+= (u64
)robj
->rdev
->mc
.vram_location
;
93 robj
->gpu_addr
+= (u64
)robj
->rdev
->mc
.gtt_location
;
96 DRM_ERROR("Unknown placement %d\n", robj
->tobj
.mem
.mem_type
);
97 robj
->gpu_addr
= 0xFFFFFFFFFFFFFFFFULL
;
102 static inline uint32_t radeon_object_flags_from_domain(uint32_t domain
)
105 if (domain
& RADEON_GEM_DOMAIN_VRAM
) {
106 flags
|= TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_WC
| TTM_PL_FLAG_UNCACHED
;
108 if (domain
& RADEON_GEM_DOMAIN_GTT
) {
109 flags
|= TTM_PL_FLAG_TT
| TTM_PL_MASK_CACHING
;
111 if (domain
& RADEON_GEM_DOMAIN_CPU
) {
112 flags
|= TTM_PL_FLAG_SYSTEM
| TTM_PL_MASK_CACHING
;
115 flags
|= TTM_PL_FLAG_SYSTEM
| TTM_PL_MASK_CACHING
;
120 int radeon_object_create(struct radeon_device
*rdev
,
121 struct drm_gem_object
*gobj
,
126 struct radeon_object
**robj_ptr
)
128 struct radeon_object
*robj
;
129 enum ttm_bo_type type
;
133 if (unlikely(rdev
->mman
.bdev
.dev_mapping
== NULL
)) {
134 rdev
->mman
.bdev
.dev_mapping
= rdev
->ddev
->dev_mapping
;
137 type
= ttm_bo_type_kernel
;
139 type
= ttm_bo_type_device
;
142 robj
= kzalloc(sizeof(struct radeon_object
), GFP_KERNEL
);
148 robj
->surface_reg
= -1;
149 INIT_LIST_HEAD(&robj
->list
);
151 flags
= radeon_object_flags_from_domain(domain
);
152 r
= ttm_buffer_object_init(&rdev
->mman
.bdev
, &robj
->tobj
, size
, type
, flags
,
153 0, 0, false, NULL
, size
,
154 &radeon_ttm_object_object_destroy
);
155 if (unlikely(r
!= 0)) {
156 /* ttm call radeon_ttm_object_object_destroy if error happen */
157 DRM_ERROR("Failed to allocate TTM object (%ld, 0x%08X, %u)\n",
163 list_add_tail(&robj
->list
, &rdev
->gem
.objects
);
168 int radeon_object_kmap(struct radeon_object
*robj
, void **ptr
)
172 spin_lock(&robj
->tobj
.lock
);
177 spin_unlock(&robj
->tobj
.lock
);
180 spin_unlock(&robj
->tobj
.lock
);
181 r
= ttm_bo_kmap(&robj
->tobj
, 0, robj
->tobj
.num_pages
, &robj
->kmap
);
185 spin_lock(&robj
->tobj
.lock
);
186 robj
->kptr
= ttm_kmap_obj_virtual(&robj
->kmap
, &robj
->is_iomem
);
187 spin_unlock(&robj
->tobj
.lock
);
191 radeon_object_check_tiling(robj
, 0, 0);
195 void radeon_object_kunmap(struct radeon_object
*robj
)
197 spin_lock(&robj
->tobj
.lock
);
198 if (robj
->kptr
== NULL
) {
199 spin_unlock(&robj
->tobj
.lock
);
203 spin_unlock(&robj
->tobj
.lock
);
204 radeon_object_check_tiling(robj
, 0, 0);
205 ttm_bo_kunmap(&robj
->kmap
);
208 void radeon_object_unref(struct radeon_object
**robj
)
210 struct ttm_buffer_object
*tobj
;
212 if ((*robj
) == NULL
) {
215 tobj
= &((*robj
)->tobj
);
222 int radeon_object_mmap(struct radeon_object
*robj
, uint64_t *offset
)
224 *offset
= robj
->tobj
.addr_space_offset
;
228 int radeon_object_pin(struct radeon_object
*robj
, uint32_t domain
,
235 flags
= radeon_object_flags_from_domain(domain
);
236 spin_lock(&robj
->tobj
.lock
);
237 if (robj
->pin_count
) {
239 if (gpu_addr
!= NULL
) {
240 *gpu_addr
= robj
->gpu_addr
;
242 spin_unlock(&robj
->tobj
.lock
);
245 spin_unlock(&robj
->tobj
.lock
);
246 r
= radeon_object_reserve(robj
, false);
247 if (unlikely(r
!= 0)) {
248 DRM_ERROR("radeon: failed to reserve object for pinning it.\n");
251 tmp
= robj
->tobj
.mem
.placement
;
252 ttm_flag_masked(&tmp
, flags
, TTM_PL_MASK_MEM
);
253 robj
->tobj
.proposed_placement
= tmp
| TTM_PL_FLAG_NO_EVICT
| TTM_PL_MASK_CACHING
;
254 r
= ttm_buffer_object_validate(&robj
->tobj
,
255 robj
->tobj
.proposed_placement
,
257 radeon_object_gpu_addr(robj
);
258 if (gpu_addr
!= NULL
) {
259 *gpu_addr
= robj
->gpu_addr
;
262 if (unlikely(r
!= 0)) {
263 DRM_ERROR("radeon: failed to pin object.\n");
265 radeon_object_unreserve(robj
);
269 void radeon_object_unpin(struct radeon_object
*robj
)
274 spin_lock(&robj
->tobj
.lock
);
275 if (!robj
->pin_count
) {
276 spin_unlock(&robj
->tobj
.lock
);
277 printk(KERN_WARNING
"Unpin not necessary for %p !\n", robj
);
281 if (robj
->pin_count
) {
282 spin_unlock(&robj
->tobj
.lock
);
285 spin_unlock(&robj
->tobj
.lock
);
286 r
= radeon_object_reserve(robj
, false);
287 if (unlikely(r
!= 0)) {
288 DRM_ERROR("radeon: failed to reserve object for unpinning it.\n");
291 flags
= robj
->tobj
.mem
.placement
;
292 robj
->tobj
.proposed_placement
= flags
& ~TTM_PL_FLAG_NO_EVICT
;
293 r
= ttm_buffer_object_validate(&robj
->tobj
,
294 robj
->tobj
.proposed_placement
,
296 if (unlikely(r
!= 0)) {
297 DRM_ERROR("radeon: failed to unpin buffer.\n");
299 radeon_object_unreserve(robj
);
302 int radeon_object_wait(struct radeon_object
*robj
)
306 /* FIXME: should use block reservation instead */
307 r
= radeon_object_reserve(robj
, true);
308 if (unlikely(r
!= 0)) {
309 DRM_ERROR("radeon: failed to reserve object for waiting.\n");
312 spin_lock(&robj
->tobj
.lock
);
313 if (robj
->tobj
.sync_obj
) {
314 r
= ttm_bo_wait(&robj
->tobj
, true, true, false);
316 spin_unlock(&robj
->tobj
.lock
);
317 radeon_object_unreserve(robj
);
321 int radeon_object_busy_domain(struct radeon_object
*robj
, uint32_t *cur_placement
)
325 r
= radeon_object_reserve(robj
, true);
326 if (unlikely(r
!= 0)) {
327 DRM_ERROR("radeon: failed to reserve object for waiting.\n");
330 spin_lock(&robj
->tobj
.lock
);
331 *cur_placement
= robj
->tobj
.mem
.mem_type
;
332 if (robj
->tobj
.sync_obj
) {
333 r
= ttm_bo_wait(&robj
->tobj
, true, true, true);
335 spin_unlock(&robj
->tobj
.lock
);
336 radeon_object_unreserve(robj
);
340 int radeon_object_evict_vram(struct radeon_device
*rdev
)
342 if (rdev
->flags
& RADEON_IS_IGP
) {
343 /* Useless to evict on IGP chips */
346 return ttm_bo_evict_mm(&rdev
->mman
.bdev
, TTM_PL_VRAM
);
349 void radeon_object_force_delete(struct radeon_device
*rdev
)
351 struct radeon_object
*robj
, *n
;
352 struct drm_gem_object
*gobj
;
354 if (list_empty(&rdev
->gem
.objects
)) {
357 DRM_ERROR("Userspace still has active objects !\n");
358 list_for_each_entry_safe(robj
, n
, &rdev
->gem
.objects
, list
) {
359 mutex_lock(&rdev
->ddev
->struct_mutex
);
361 DRM_ERROR("Force free for (%p,%p,%lu,%lu)\n",
362 gobj
, robj
, (unsigned long)gobj
->size
,
363 *((unsigned long *)&gobj
->refcount
));
364 list_del_init(&robj
->list
);
365 radeon_object_unref(&robj
);
366 gobj
->driver_private
= NULL
;
367 drm_gem_object_unreference(gobj
);
368 mutex_unlock(&rdev
->ddev
->struct_mutex
);
372 int radeon_object_init(struct radeon_device
*rdev
)
374 /* Add an MTRR for the VRAM */
375 rdev
->mc
.vram_mtrr
= mtrr_add(rdev
->mc
.aper_base
, rdev
->mc
.aper_size
,
376 MTRR_TYPE_WRCOMB
, 1);
377 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
378 rdev
->mc
.mc_vram_size
>> 20,
379 (unsigned long long)rdev
->mc
.aper_size
>> 20);
380 DRM_INFO("RAM width %dbits %cDR\n",
381 rdev
->mc
.vram_width
, rdev
->mc
.vram_is_ddr
? 'D' : 'S');
382 return radeon_ttm_init(rdev
);
385 void radeon_object_fini(struct radeon_device
*rdev
)
387 radeon_ttm_fini(rdev
);
390 void radeon_object_list_add_object(struct radeon_object_list
*lobj
,
391 struct list_head
*head
)
394 list_add(&lobj
->list
, head
);
396 list_add_tail(&lobj
->list
, head
);
400 int radeon_object_list_reserve(struct list_head
*head
)
402 struct radeon_object_list
*lobj
;
406 list_for_each(i
, head
) {
407 lobj
= list_entry(i
, struct radeon_object_list
, list
);
408 if (!lobj
->robj
->pin_count
) {
409 r
= radeon_object_reserve(lobj
->robj
, true);
410 if (unlikely(r
!= 0)) {
411 DRM_ERROR("radeon: failed to reserve object.\n");
420 void radeon_object_list_unreserve(struct list_head
*head
)
422 struct radeon_object_list
*lobj
;
425 list_for_each(i
, head
) {
426 lobj
= list_entry(i
, struct radeon_object_list
, list
);
427 if (!lobj
->robj
->pin_count
) {
428 radeon_object_unreserve(lobj
->robj
);
434 int radeon_object_list_validate(struct list_head
*head
, void *fence
)
436 struct radeon_object_list
*lobj
;
437 struct radeon_object
*robj
;
438 struct radeon_fence
*old_fence
= NULL
;
442 r
= radeon_object_list_reserve(head
);
443 if (unlikely(r
!= 0)) {
444 radeon_object_list_unreserve(head
);
447 list_for_each(i
, head
) {
448 lobj
= list_entry(i
, struct radeon_object_list
, list
);
450 if (!robj
->pin_count
) {
452 robj
->tobj
.proposed_placement
=
453 radeon_object_flags_from_domain(lobj
->wdomain
);
455 robj
->tobj
.proposed_placement
=
456 radeon_object_flags_from_domain(lobj
->rdomain
);
458 r
= ttm_buffer_object_validate(&robj
->tobj
,
459 robj
->tobj
.proposed_placement
,
462 DRM_ERROR("radeon: failed to validate.\n");
465 radeon_object_gpu_addr(robj
);
467 lobj
->gpu_offset
= robj
->gpu_addr
;
468 lobj
->tiling_flags
= robj
->tiling_flags
;
470 old_fence
= (struct radeon_fence
*)robj
->tobj
.sync_obj
;
471 robj
->tobj
.sync_obj
= radeon_fence_ref(fence
);
472 robj
->tobj
.sync_obj_arg
= NULL
;
475 radeon_fence_unref(&old_fence
);
481 void radeon_object_list_unvalidate(struct list_head
*head
)
483 struct radeon_object_list
*lobj
;
484 struct radeon_fence
*old_fence
= NULL
;
487 list_for_each(i
, head
) {
488 lobj
= list_entry(i
, struct radeon_object_list
, list
);
489 old_fence
= (struct radeon_fence
*)lobj
->robj
->tobj
.sync_obj
;
490 lobj
->robj
->tobj
.sync_obj
= NULL
;
492 radeon_fence_unref(&old_fence
);
495 radeon_object_list_unreserve(head
);
498 void radeon_object_list_clean(struct list_head
*head
)
500 radeon_object_list_unreserve(head
);
503 int radeon_object_fbdev_mmap(struct radeon_object
*robj
,
504 struct vm_area_struct
*vma
)
506 return ttm_fbdev_mmap(vma
, &robj
->tobj
);
509 unsigned long radeon_object_size(struct radeon_object
*robj
)
511 return robj
->tobj
.num_pages
<< PAGE_SHIFT
;
514 int radeon_object_get_surface_reg(struct radeon_object
*robj
)
516 struct radeon_device
*rdev
= robj
->rdev
;
517 struct radeon_surface_reg
*reg
;
518 struct radeon_object
*old_object
;
522 if (!robj
->tiling_flags
)
525 if (robj
->surface_reg
>= 0) {
526 reg
= &rdev
->surface_regs
[robj
->surface_reg
];
527 i
= robj
->surface_reg
;
532 for (i
= 0; i
< RADEON_GEM_MAX_SURFACES
; i
++) {
534 reg
= &rdev
->surface_regs
[i
];
538 old_object
= reg
->robj
;
539 if (old_object
->pin_count
== 0)
543 /* if we are all out */
544 if (i
== RADEON_GEM_MAX_SURFACES
) {
547 /* find someone with a surface reg and nuke their BO */
548 reg
= &rdev
->surface_regs
[steal
];
549 old_object
= reg
->robj
;
550 /* blow away the mapping */
551 DRM_DEBUG("stealing surface reg %d from %p\n", steal
, old_object
);
552 ttm_bo_unmap_virtual(&old_object
->tobj
);
553 old_object
->surface_reg
= -1;
557 robj
->surface_reg
= i
;
561 radeon_set_surface_reg(rdev
, i
, robj
->tiling_flags
, robj
->pitch
,
562 robj
->tobj
.mem
.mm_node
->start
<< PAGE_SHIFT
,
563 robj
->tobj
.num_pages
<< PAGE_SHIFT
);
567 void radeon_object_clear_surface_reg(struct radeon_object
*robj
)
569 struct radeon_device
*rdev
= robj
->rdev
;
570 struct radeon_surface_reg
*reg
;
572 if (robj
->surface_reg
== -1)
575 reg
= &rdev
->surface_regs
[robj
->surface_reg
];
576 radeon_clear_surface_reg(rdev
, robj
->surface_reg
);
579 robj
->surface_reg
= -1;
582 void radeon_object_set_tiling_flags(struct radeon_object
*robj
,
583 uint32_t tiling_flags
, uint32_t pitch
)
585 robj
->tiling_flags
= tiling_flags
;
589 void radeon_object_get_tiling_flags(struct radeon_object
*robj
,
590 uint32_t *tiling_flags
,
594 *tiling_flags
= robj
->tiling_flags
;
596 *pitch
= robj
->pitch
;
599 int radeon_object_check_tiling(struct radeon_object
*robj
, bool has_moved
,
602 if (!(robj
->tiling_flags
& RADEON_TILING_SURFACE
))
606 radeon_object_clear_surface_reg(robj
);
610 if (robj
->tobj
.mem
.mem_type
!= TTM_PL_VRAM
) {
614 if (robj
->surface_reg
>= 0)
615 radeon_object_clear_surface_reg(robj
);
619 if ((robj
->surface_reg
>= 0) && !has_moved
)
622 return radeon_object_get_surface_reg(robj
);
625 void radeon_bo_move_notify(struct ttm_buffer_object
*bo
,
626 struct ttm_mem_reg
*mem
)
628 struct radeon_object
*robj
= container_of(bo
, struct radeon_object
, tobj
);
629 radeon_object_check_tiling(robj
, 0, 1);
632 void radeon_bo_fault_reserve_notify(struct ttm_buffer_object
*bo
)
634 struct radeon_object
*robj
= container_of(bo
, struct radeon_object
, tobj
);
635 radeon_object_check_tiling(robj
, 0, 0);