2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
34 * Most engines on the GPU are fed via ring buffers. Ring
35 * buffers are areas of GPU accessible memory that the host
36 * writes commands into and the GPU reads commands out of.
37 * There is a rptr (read pointer) that determines where the
38 * GPU is currently reading, and a wptr (write pointer)
39 * which determines where the host has written. When the
40 * pointers are equal, the ring is idle. When the host
41 * writes commands to the ring buffer, it increments the
42 * wptr. The GPU then starts fetching commands and executes
43 * them until the pointers are equal again.
45 static int radeon_debugfs_ring_init(struct radeon_device
*rdev
, struct radeon_ring
*ring
);
48 * radeon_ring_write - write a value to the ring
50 * @ring: radeon_ring structure holding ring information
51 * @v: dword (dw) value to write
53 * Write a value to the requested ring buffer (all asics).
55 void radeon_ring_write(struct radeon_ring
*ring
, uint32_t v
)
58 if (ring
->count_dw
<= 0) {
59 DRM_ERROR("radeon: writing more dwords to the ring than expected!\n");
62 ring
->ring
[ring
->wptr
++] = v
;
63 ring
->wptr
&= ring
->ptr_mask
;
69 * radeon_ring_supports_scratch_reg - check if the ring supports
70 * writing to scratch registers
72 * @rdev: radeon_device pointer
73 * @ring: radeon_ring structure holding ring information
75 * Check if a specific ring supports writing to scratch registers (all asics).
76 * Returns true if the ring supports writing to scratch regs, false if not.
78 bool radeon_ring_supports_scratch_reg(struct radeon_device
*rdev
,
79 struct radeon_ring
*ring
)
82 case RADEON_RING_TYPE_GFX_INDEX
:
83 case CAYMAN_RING_TYPE_CP1_INDEX
:
84 case CAYMAN_RING_TYPE_CP2_INDEX
:
92 * radeon_ring_free_size - update the free size
94 * @rdev: radeon_device pointer
95 * @ring: radeon_ring structure holding ring information
97 * Update the free dw slots in the ring buffer (all asics).
99 void radeon_ring_free_size(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
101 uint32_t rptr
= radeon_ring_get_rptr(rdev
, ring
);
103 /* This works because ring_size is a power of 2 */
104 ring
->ring_free_dw
= rptr
+ (ring
->ring_size
/ 4);
105 ring
->ring_free_dw
-= ring
->wptr
;
106 ring
->ring_free_dw
&= ring
->ptr_mask
;
107 if (!ring
->ring_free_dw
) {
108 /* this is an empty ring */
109 ring
->ring_free_dw
= ring
->ring_size
/ 4;
110 /* update lockup info to avoid false positive */
111 radeon_ring_lockup_update(rdev
, ring
);
116 * radeon_ring_alloc - allocate space on the ring buffer
118 * @rdev: radeon_device pointer
119 * @ring: radeon_ring structure holding ring information
120 * @ndw: number of dwords to allocate in the ring buffer
122 * Allocate @ndw dwords in the ring buffer (all asics).
123 * Returns 0 on success, error on failure.
125 int radeon_ring_alloc(struct radeon_device
*rdev
, struct radeon_ring
*ring
, unsigned ndw
)
129 /* make sure we aren't trying to allocate more space than there is on the ring */
130 if (ndw
> (ring
->ring_size
/ 4))
132 /* Align requested size with padding so unlock_commit can
134 radeon_ring_free_size(rdev
, ring
);
135 ndw
= (ndw
+ ring
->align_mask
) & ~ring
->align_mask
;
136 while (ndw
> (ring
->ring_free_dw
- 1)) {
137 radeon_ring_free_size(rdev
, ring
);
138 if (ndw
< ring
->ring_free_dw
) {
141 r
= radeon_fence_wait_next(rdev
, ring
->idx
);
145 ring
->count_dw
= ndw
;
146 ring
->wptr_old
= ring
->wptr
;
151 * radeon_ring_lock - lock the ring and allocate space on it
153 * @rdev: radeon_device pointer
154 * @ring: radeon_ring structure holding ring information
155 * @ndw: number of dwords to allocate in the ring buffer
157 * Lock the ring and allocate @ndw dwords in the ring buffer
159 * Returns 0 on success, error on failure.
161 int radeon_ring_lock(struct radeon_device
*rdev
, struct radeon_ring
*ring
, unsigned ndw
)
165 mutex_lock(&rdev
->ring_lock
);
166 r
= radeon_ring_alloc(rdev
, ring
, ndw
);
168 mutex_unlock(&rdev
->ring_lock
);
175 * radeon_ring_commit - tell the GPU to execute the new
176 * commands on the ring buffer
178 * @rdev: radeon_device pointer
179 * @ring: radeon_ring structure holding ring information
181 * Update the wptr (write pointer) to tell the GPU to
182 * execute new commands on the ring buffer (all asics).
184 void radeon_ring_commit(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
186 /* If we are emitting the HDP flush via the ring buffer, we need to
187 * do it before padding.
189 if (rdev
->asic
->ring
[ring
->idx
]->hdp_flush
)
190 rdev
->asic
->ring
[ring
->idx
]->hdp_flush(rdev
, ring
);
191 /* We pad to match fetch size */
192 while (ring
->wptr
& ring
->align_mask
) {
193 radeon_ring_write(ring
, ring
->nop
);
196 /* If we are emitting the HDP flush via MMIO, we need to do it after
197 * all CPU writes to VRAM finished.
199 if (rdev
->asic
->mmio_hdp_flush
)
200 rdev
->asic
->mmio_hdp_flush(rdev
);
201 radeon_ring_set_wptr(rdev
, ring
);
205 * radeon_ring_unlock_commit - tell the GPU to execute the new
206 * commands on the ring buffer and unlock it
208 * @rdev: radeon_device pointer
209 * @ring: radeon_ring structure holding ring information
211 * Call radeon_ring_commit() then unlock the ring (all asics).
213 void radeon_ring_unlock_commit(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
215 radeon_ring_commit(rdev
, ring
);
216 mutex_unlock(&rdev
->ring_lock
);
220 * radeon_ring_undo - reset the wptr
222 * @ring: radeon_ring structure holding ring information
224 * Reset the driver's copy of the wptr (all asics).
226 void radeon_ring_undo(struct radeon_ring
*ring
)
228 ring
->wptr
= ring
->wptr_old
;
232 * radeon_ring_unlock_undo - reset the wptr and unlock the ring
234 * @ring: radeon_ring structure holding ring information
236 * Call radeon_ring_undo() then unlock the ring (all asics).
238 void radeon_ring_unlock_undo(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
240 radeon_ring_undo(ring
);
241 mutex_unlock(&rdev
->ring_lock
);
245 * radeon_ring_lockup_update - update lockup variables
247 * @ring: radeon_ring structure holding ring information
249 * Update the last rptr value and timestamp (all asics).
251 void radeon_ring_lockup_update(struct radeon_device
*rdev
,
252 struct radeon_ring
*ring
)
254 atomic_set(&ring
->last_rptr
, radeon_ring_get_rptr(rdev
, ring
));
255 atomic64_set(&ring
->last_activity
, jiffies_64
);
259 * radeon_ring_test_lockup() - check if ring is lockedup by recording information
260 * @rdev: radeon device structure
261 * @ring: radeon_ring structure holding ring information
264 bool radeon_ring_test_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
266 uint32_t rptr
= radeon_ring_get_rptr(rdev
, ring
);
267 uint64_t last
= atomic64_read(&ring
->last_activity
);
270 if (rptr
!= atomic_read(&ring
->last_rptr
)) {
271 /* ring is still working, no lockup */
272 radeon_ring_lockup_update(rdev
, ring
);
276 elapsed
= jiffies_to_msecs(jiffies_64
- last
);
277 if (radeon_lockup_timeout
&& elapsed
>= radeon_lockup_timeout
) {
278 dev_err(rdev
->dev
, "ring %d stalled for more than %llumsec\n",
282 /* give a chance to the GPU ... */
287 * radeon_ring_backup - Back up the content of a ring
289 * @rdev: radeon_device pointer
290 * @ring: the ring we want to back up
292 * Saves all unprocessed commits from a ring, returns the number of dwords saved.
294 unsigned radeon_ring_backup(struct radeon_device
*rdev
, struct radeon_ring
*ring
,
297 unsigned size
, ptr
, i
;
299 /* just in case lock the ring */
300 mutex_lock(&rdev
->ring_lock
);
303 if (ring
->ring_obj
== NULL
) {
304 mutex_unlock(&rdev
->ring_lock
);
308 /* it doesn't make sense to save anything if all fences are signaled */
309 if (!radeon_fence_count_emitted(rdev
, ring
->idx
)) {
310 mutex_unlock(&rdev
->ring_lock
);
314 /* calculate the number of dw on the ring */
315 if (ring
->rptr_save_reg
)
316 ptr
= RREG32(ring
->rptr_save_reg
);
317 else if (rdev
->wb
.enabled
)
318 ptr
= le32_to_cpu(*ring
->next_rptr_cpu_addr
);
320 /* no way to read back the next rptr */
321 mutex_unlock(&rdev
->ring_lock
);
325 size
= ring
->wptr
+ (ring
->ring_size
/ 4);
327 size
&= ring
->ptr_mask
;
329 mutex_unlock(&rdev
->ring_lock
);
333 /* and then save the content of the ring */
334 *data
= kmalloc_array(size
, sizeof(uint32_t), GFP_KERNEL
);
336 mutex_unlock(&rdev
->ring_lock
);
339 for (i
= 0; i
< size
; ++i
) {
340 (*data
)[i
] = ring
->ring
[ptr
++];
341 ptr
&= ring
->ptr_mask
;
344 mutex_unlock(&rdev
->ring_lock
);
349 * radeon_ring_restore - append saved commands to the ring again
351 * @rdev: radeon_device pointer
352 * @ring: ring to append commands to
353 * @size: number of dwords we want to write
354 * @data: saved commands
356 * Allocates space on the ring and restore the previously saved commands.
358 int radeon_ring_restore(struct radeon_device
*rdev
, struct radeon_ring
*ring
,
359 unsigned size
, uint32_t *data
)
366 /* restore the saved ring content */
367 r
= radeon_ring_lock(rdev
, ring
, size
);
371 for (i
= 0; i
< size
; ++i
) {
372 radeon_ring_write(ring
, data
[i
]);
375 radeon_ring_unlock_commit(rdev
, ring
);
381 * radeon_ring_init - init driver ring struct.
383 * @rdev: radeon_device pointer
384 * @ring: radeon_ring structure holding ring information
385 * @ring_size: size of the ring
386 * @rptr_offs: offset of the rptr writeback location in the WB buffer
387 * @nop: nop packet for this ring
389 * Initialize the driver information for the selected ring (all asics).
390 * Returns 0 on success, error on failure.
392 int radeon_ring_init(struct radeon_device
*rdev
, struct radeon_ring
*ring
, unsigned ring_size
,
393 unsigned rptr_offs
, u32 nop
)
397 ring
->ring_size
= ring_size
;
398 ring
->rptr_offs
= rptr_offs
;
400 /* Allocate ring buffer */
401 if (ring
->ring_obj
== NULL
) {
402 r
= radeon_bo_create(rdev
, ring
->ring_size
, PAGE_SIZE
, true,
403 RADEON_GEM_DOMAIN_GTT
,
404 (rdev
->flags
& RADEON_IS_PCIE
) ?
405 RADEON_GEM_GTT_WC
: 0,
406 NULL
, &ring
->ring_obj
);
408 dev_err(rdev
->dev
, "(%d) ring create failed\n", r
);
411 r
= radeon_bo_reserve(ring
->ring_obj
, false);
412 if (unlikely(r
!= 0))
414 r
= radeon_bo_pin(ring
->ring_obj
, RADEON_GEM_DOMAIN_GTT
,
417 radeon_bo_unreserve(ring
->ring_obj
);
418 dev_err(rdev
->dev
, "(%d) ring pin failed\n", r
);
421 r
= radeon_bo_kmap(ring
->ring_obj
,
422 (void **)&ring
->ring
);
423 radeon_bo_unreserve(ring
->ring_obj
);
425 dev_err(rdev
->dev
, "(%d) ring map failed\n", r
);
429 ring
->ptr_mask
= (ring
->ring_size
/ 4) - 1;
430 ring
->ring_free_dw
= ring
->ring_size
/ 4;
431 if (rdev
->wb
.enabled
) {
432 u32 index
= RADEON_WB_RING0_NEXT_RPTR
+ (ring
->idx
* 4);
433 ring
->next_rptr_gpu_addr
= rdev
->wb
.gpu_addr
+ index
;
434 ring
->next_rptr_cpu_addr
= &rdev
->wb
.wb
[index
/4];
436 if (radeon_debugfs_ring_init(rdev
, ring
)) {
437 DRM_ERROR("Failed to register debugfs file for rings !\n");
439 radeon_ring_lockup_update(rdev
, ring
);
444 * radeon_ring_fini - tear down the driver ring struct.
446 * @rdev: radeon_device pointer
447 * @ring: radeon_ring structure holding ring information
449 * Tear down the driver information for the selected ring (all asics).
451 void radeon_ring_fini(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
454 struct radeon_bo
*ring_obj
;
456 mutex_lock(&rdev
->ring_lock
);
457 ring_obj
= ring
->ring_obj
;
460 ring
->ring_obj
= NULL
;
461 mutex_unlock(&rdev
->ring_lock
);
464 r
= radeon_bo_reserve(ring_obj
, false);
465 if (likely(r
== 0)) {
466 radeon_bo_kunmap(ring_obj
);
467 radeon_bo_unpin(ring_obj
);
468 radeon_bo_unreserve(ring_obj
);
470 radeon_bo_unref(&ring_obj
);
477 #if defined(CONFIG_DEBUG_FS)
479 static int radeon_debugfs_ring_info(struct seq_file
*m
, void *data
)
481 struct drm_info_node
*node
= (struct drm_info_node
*) m
->private;
482 struct drm_device
*dev
= node
->minor
->dev
;
483 struct radeon_device
*rdev
= dev
->dev_private
;
484 int ridx
= *(int*)node
->info_ent
->data
;
485 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
487 uint32_t rptr
, wptr
, rptr_next
;
488 unsigned count
, i
, j
;
490 radeon_ring_free_size(rdev
, ring
);
491 count
= (ring
->ring_size
/ 4) - ring
->ring_free_dw
;
493 wptr
= radeon_ring_get_wptr(rdev
, ring
);
494 seq_printf(m
, "wptr: 0x%08x [%5d]\n",
497 rptr
= radeon_ring_get_rptr(rdev
, ring
);
498 seq_printf(m
, "rptr: 0x%08x [%5d]\n",
501 if (ring
->rptr_save_reg
) {
502 rptr_next
= RREG32(ring
->rptr_save_reg
);
503 seq_printf(m
, "rptr next(0x%04x): 0x%08x [%5d]\n",
504 ring
->rptr_save_reg
, rptr_next
, rptr_next
);
508 seq_printf(m
, "driver's copy of the wptr: 0x%08x [%5d]\n",
509 ring
->wptr
, ring
->wptr
);
510 seq_printf(m
, "last semaphore signal addr : 0x%016llx\n",
511 ring
->last_semaphore_signal_addr
);
512 seq_printf(m
, "last semaphore wait addr : 0x%016llx\n",
513 ring
->last_semaphore_wait_addr
);
514 seq_printf(m
, "%u free dwords in ring\n", ring
->ring_free_dw
);
515 seq_printf(m
, "%u dwords in ring\n", count
);
520 /* print 8 dw before current rptr as often it's the last executed
521 * packet that is the root issue
523 i
= (rptr
+ ring
->ptr_mask
+ 1 - 32) & ring
->ptr_mask
;
524 for (j
= 0; j
<= (count
+ 32); j
++) {
525 seq_printf(m
, "r[%5d]=0x%08x", i
, ring
->ring
[i
]);
531 i
= (i
+ 1) & ring
->ptr_mask
;
536 static int radeon_gfx_index
= RADEON_RING_TYPE_GFX_INDEX
;
537 static int cayman_cp1_index
= CAYMAN_RING_TYPE_CP1_INDEX
;
538 static int cayman_cp2_index
= CAYMAN_RING_TYPE_CP2_INDEX
;
539 static int radeon_dma1_index
= R600_RING_TYPE_DMA_INDEX
;
540 static int radeon_dma2_index
= CAYMAN_RING_TYPE_DMA1_INDEX
;
541 static int r600_uvd_index
= R600_RING_TYPE_UVD_INDEX
;
542 static int si_vce1_index
= TN_RING_TYPE_VCE1_INDEX
;
543 static int si_vce2_index
= TN_RING_TYPE_VCE2_INDEX
;
545 static struct drm_info_list radeon_debugfs_ring_info_list
[] = {
546 {"radeon_ring_gfx", radeon_debugfs_ring_info
, 0, &radeon_gfx_index
},
547 {"radeon_ring_cp1", radeon_debugfs_ring_info
, 0, &cayman_cp1_index
},
548 {"radeon_ring_cp2", radeon_debugfs_ring_info
, 0, &cayman_cp2_index
},
549 {"radeon_ring_dma1", radeon_debugfs_ring_info
, 0, &radeon_dma1_index
},
550 {"radeon_ring_dma2", radeon_debugfs_ring_info
, 0, &radeon_dma2_index
},
551 {"radeon_ring_uvd", radeon_debugfs_ring_info
, 0, &r600_uvd_index
},
552 {"radeon_ring_vce1", radeon_debugfs_ring_info
, 0, &si_vce1_index
},
553 {"radeon_ring_vce2", radeon_debugfs_ring_info
, 0, &si_vce2_index
},
558 static int radeon_debugfs_ring_init(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
560 #if defined(CONFIG_DEBUG_FS)
562 for (i
= 0; i
< ARRAY_SIZE(radeon_debugfs_ring_info_list
); ++i
) {
563 struct drm_info_list
*info
= &radeon_debugfs_ring_info_list
[i
];
564 int ridx
= *(int*)radeon_debugfs_ring_info_list
[i
].data
;
567 if (&rdev
->ring
[ridx
] != ring
)
570 r
= radeon_debugfs_add_files(rdev
, info
, 1);