2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
29 #include <linux/seq_file.h>
30 #include <linux/slab.h>
32 #include <drm/amdgpu_drm.h>
38 * Most engines on the GPU are fed via ring buffers. Ring
39 * buffers are areas of GPU accessible memory that the host
40 * writes commands into and the GPU reads commands out of.
41 * There is a rptr (read pointer) that determines where the
42 * GPU is currently reading, and a wptr (write pointer)
43 * which determines where the host has written. When the
44 * pointers are equal, the ring is idle. When the host
45 * writes commands to the ring buffer, it increments the
46 * wptr. The GPU then starts fetching commands and executes
47 * them until the pointers are equal again.
49 static int amdgpu_debugfs_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
);
52 * amdgpu_ring_free_size - update the free size
54 * @adev: amdgpu_device pointer
55 * @ring: amdgpu_ring structure holding ring information
57 * Update the free dw slots in the ring buffer (all asics).
59 void amdgpu_ring_free_size(struct amdgpu_ring
*ring
)
61 uint32_t rptr
= amdgpu_ring_get_rptr(ring
);
63 /* This works because ring_size is a power of 2 */
64 ring
->ring_free_dw
= rptr
+ (ring
->ring_size
/ 4);
65 ring
->ring_free_dw
-= ring
->wptr
;
66 ring
->ring_free_dw
&= ring
->ptr_mask
;
67 if (!ring
->ring_free_dw
) {
68 /* this is an empty ring */
69 ring
->ring_free_dw
= ring
->ring_size
/ 4;
70 /* update lockup info to avoid false positive */
71 amdgpu_ring_lockup_update(ring
);
76 * amdgpu_ring_alloc - allocate space on the ring buffer
78 * @adev: amdgpu_device pointer
79 * @ring: amdgpu_ring structure holding ring information
80 * @ndw: number of dwords to allocate in the ring buffer
82 * Allocate @ndw dwords in the ring buffer (all asics).
83 * Returns 0 on success, error on failure.
85 int amdgpu_ring_alloc(struct amdgpu_ring
*ring
, unsigned ndw
)
89 /* make sure we aren't trying to allocate more space than there is on the ring */
90 if (ndw
> (ring
->ring_size
/ 4))
92 /* Align requested size with padding so unlock_commit can
94 amdgpu_ring_free_size(ring
);
95 ndw
= (ndw
+ ring
->align_mask
) & ~ring
->align_mask
;
96 while (ndw
> (ring
->ring_free_dw
- 1)) {
97 amdgpu_ring_free_size(ring
);
98 if (ndw
< ring
->ring_free_dw
) {
101 r
= amdgpu_fence_wait_next(ring
);
105 ring
->count_dw
= ndw
;
106 ring
->wptr_old
= ring
->wptr
;
111 * amdgpu_ring_lock - lock the ring and allocate space on it
113 * @adev: amdgpu_device pointer
114 * @ring: amdgpu_ring structure holding ring information
115 * @ndw: number of dwords to allocate in the ring buffer
117 * Lock the ring and allocate @ndw dwords in the ring buffer
119 * Returns 0 on success, error on failure.
121 int amdgpu_ring_lock(struct amdgpu_ring
*ring
, unsigned ndw
)
125 mutex_lock(ring
->ring_lock
);
126 r
= amdgpu_ring_alloc(ring
, ndw
);
128 mutex_unlock(ring
->ring_lock
);
134 /** amdgpu_ring_insert_nop - insert NOP packets
136 * @ring: amdgpu_ring structure holding ring information
137 * @count: the number of NOP packets to insert
139 * This is the generic insert_nop function for rings except SDMA
141 void amdgpu_ring_insert_nop(struct amdgpu_ring
*ring
, uint32_t count
)
145 for (i
= 0; i
< count
; i
++)
146 amdgpu_ring_write(ring
, ring
->nop
);
150 * amdgpu_ring_commit - tell the GPU to execute the new
151 * commands on the ring buffer
153 * @adev: amdgpu_device pointer
154 * @ring: amdgpu_ring structure holding ring information
156 * Update the wptr (write pointer) to tell the GPU to
157 * execute new commands on the ring buffer (all asics).
159 void amdgpu_ring_commit(struct amdgpu_ring
*ring
)
163 /* We pad to match fetch size */
164 count
= ring
->align_mask
+ 1 - (ring
->wptr
& ring
->align_mask
);
165 count
%= ring
->align_mask
+ 1;
166 ring
->funcs
->insert_nop(ring
, count
);
169 amdgpu_ring_set_wptr(ring
);
173 * amdgpu_ring_unlock_commit - tell the GPU to execute the new
174 * commands on the ring buffer and unlock it
176 * @ring: amdgpu_ring structure holding ring information
178 * Call amdgpu_ring_commit() then unlock the ring (all asics).
180 void amdgpu_ring_unlock_commit(struct amdgpu_ring
*ring
)
182 amdgpu_ring_commit(ring
);
183 mutex_unlock(ring
->ring_lock
);
187 * amdgpu_ring_undo - reset the wptr
189 * @ring: amdgpu_ring structure holding ring information
191 * Reset the driver's copy of the wptr (all asics).
193 void amdgpu_ring_undo(struct amdgpu_ring
*ring
)
195 ring
->wptr
= ring
->wptr_old
;
199 * amdgpu_ring_unlock_undo - reset the wptr and unlock the ring
201 * @ring: amdgpu_ring structure holding ring information
203 * Call amdgpu_ring_undo() then unlock the ring (all asics).
205 void amdgpu_ring_unlock_undo(struct amdgpu_ring
*ring
)
207 amdgpu_ring_undo(ring
);
208 mutex_unlock(ring
->ring_lock
);
212 * amdgpu_ring_lockup_update - update lockup variables
214 * @ring: amdgpu_ring structure holding ring information
216 * Update the last rptr value and timestamp (all asics).
218 void amdgpu_ring_lockup_update(struct amdgpu_ring
*ring
)
220 atomic_set(&ring
->last_rptr
, amdgpu_ring_get_rptr(ring
));
221 atomic64_set(&ring
->last_activity
, jiffies_64
);
225 * amdgpu_ring_test_lockup() - check if ring is lockedup by recording information
226 * @ring: amdgpu_ring structure holding ring information
229 bool amdgpu_ring_test_lockup(struct amdgpu_ring
*ring
)
231 uint32_t rptr
= amdgpu_ring_get_rptr(ring
);
232 uint64_t last
= atomic64_read(&ring
->last_activity
);
235 if (rptr
!= atomic_read(&ring
->last_rptr
)) {
236 /* ring is still working, no lockup */
237 amdgpu_ring_lockup_update(ring
);
241 elapsed
= jiffies_to_msecs(jiffies_64
- last
);
242 if (amdgpu_lockup_timeout
&& elapsed
>= amdgpu_lockup_timeout
) {
243 dev_err(ring
->adev
->dev
, "ring %d stalled for more than %llumsec\n",
247 /* give a chance to the GPU ... */
252 * amdgpu_ring_backup - Back up the content of a ring
254 * @ring: the ring we want to back up
256 * Saves all unprocessed commits from a ring, returns the number of dwords saved.
258 unsigned amdgpu_ring_backup(struct amdgpu_ring
*ring
,
261 unsigned size
, ptr
, i
;
263 /* just in case lock the ring */
264 mutex_lock(ring
->ring_lock
);
267 if (ring
->ring_obj
== NULL
) {
268 mutex_unlock(ring
->ring_lock
);
272 /* it doesn't make sense to save anything if all fences are signaled */
273 if (!amdgpu_fence_count_emitted(ring
)) {
274 mutex_unlock(ring
->ring_lock
);
278 ptr
= le32_to_cpu(*ring
->next_rptr_cpu_addr
);
280 size
= ring
->wptr
+ (ring
->ring_size
/ 4);
282 size
&= ring
->ptr_mask
;
284 mutex_unlock(ring
->ring_lock
);
288 /* and then save the content of the ring */
289 *data
= kmalloc_array(size
, sizeof(uint32_t), GFP_KERNEL
);
291 mutex_unlock(ring
->ring_lock
);
294 for (i
= 0; i
< size
; ++i
) {
295 (*data
)[i
] = ring
->ring
[ptr
++];
296 ptr
&= ring
->ptr_mask
;
299 mutex_unlock(ring
->ring_lock
);
304 * amdgpu_ring_restore - append saved commands to the ring again
306 * @ring: ring to append commands to
307 * @size: number of dwords we want to write
308 * @data: saved commands
310 * Allocates space on the ring and restore the previously saved commands.
312 int amdgpu_ring_restore(struct amdgpu_ring
*ring
,
313 unsigned size
, uint32_t *data
)
320 /* restore the saved ring content */
321 r
= amdgpu_ring_lock(ring
, size
);
325 for (i
= 0; i
< size
; ++i
) {
326 amdgpu_ring_write(ring
, data
[i
]);
329 amdgpu_ring_unlock_commit(ring
);
335 * amdgpu_ring_init - init driver ring struct.
337 * @adev: amdgpu_device pointer
338 * @ring: amdgpu_ring structure holding ring information
339 * @ring_size: size of the ring
340 * @nop: nop packet for this ring
342 * Initialize the driver information for the selected ring (all asics).
343 * Returns 0 on success, error on failure.
345 int amdgpu_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
,
346 unsigned ring_size
, u32 nop
, u32 align_mask
,
347 struct amdgpu_irq_src
*irq_src
, unsigned irq_type
,
348 enum amdgpu_ring_type ring_type
)
353 if (ring
->adev
== NULL
) {
354 if (adev
->num_rings
>= AMDGPU_MAX_RINGS
)
358 ring
->idx
= adev
->num_rings
++;
359 adev
->rings
[ring
->idx
] = ring
;
360 amdgpu_fence_driver_init_ring(ring
);
363 init_waitqueue_head(&ring
->fence_drv
.fence_queue
);
365 r
= amdgpu_wb_get(adev
, &ring
->rptr_offs
);
367 dev_err(adev
->dev
, "(%d) ring rptr_offs wb alloc failed\n", r
);
371 r
= amdgpu_wb_get(adev
, &ring
->wptr_offs
);
373 dev_err(adev
->dev
, "(%d) ring wptr_offs wb alloc failed\n", r
);
377 r
= amdgpu_wb_get(adev
, &ring
->fence_offs
);
379 dev_err(adev
->dev
, "(%d) ring fence_offs wb alloc failed\n", r
);
383 r
= amdgpu_wb_get(adev
, &ring
->next_rptr_offs
);
385 dev_err(adev
->dev
, "(%d) ring next_rptr wb alloc failed\n", r
);
388 ring
->next_rptr_gpu_addr
= adev
->wb
.gpu_addr
+ (ring
->next_rptr_offs
* 4);
389 ring
->next_rptr_cpu_addr
= &adev
->wb
.wb
[ring
->next_rptr_offs
];
390 spin_lock_init(&ring
->fence_lock
);
391 r
= amdgpu_fence_driver_start_ring(ring
, irq_src
, irq_type
);
393 dev_err(adev
->dev
, "failed initializing fences (%d).\n", r
);
397 ring
->ring_lock
= &adev
->ring_lock
;
398 /* Align ring size */
399 rb_bufsz
= order_base_2(ring_size
/ 8);
400 ring_size
= (1 << (rb_bufsz
+ 1)) * 4;
401 ring
->ring_size
= ring_size
;
402 ring
->align_mask
= align_mask
;
404 ring
->type
= ring_type
;
406 /* Allocate ring buffer */
407 if (ring
->ring_obj
== NULL
) {
408 r
= amdgpu_bo_create(adev
, ring
->ring_size
, PAGE_SIZE
, true,
409 AMDGPU_GEM_DOMAIN_GTT
, 0,
410 NULL
, &ring
->ring_obj
);
412 dev_err(adev
->dev
, "(%d) ring create failed\n", r
);
415 r
= amdgpu_bo_reserve(ring
->ring_obj
, false);
416 if (unlikely(r
!= 0))
418 r
= amdgpu_bo_pin(ring
->ring_obj
, AMDGPU_GEM_DOMAIN_GTT
,
421 amdgpu_bo_unreserve(ring
->ring_obj
);
422 dev_err(adev
->dev
, "(%d) ring pin failed\n", r
);
425 r
= amdgpu_bo_kmap(ring
->ring_obj
,
426 (void **)&ring
->ring
);
427 amdgpu_bo_unreserve(ring
->ring_obj
);
429 dev_err(adev
->dev
, "(%d) ring map failed\n", r
);
433 ring
->ptr_mask
= (ring
->ring_size
/ 4) - 1;
434 ring
->ring_free_dw
= ring
->ring_size
/ 4;
436 if (amdgpu_debugfs_ring_init(adev
, ring
)) {
437 DRM_ERROR("Failed to register debugfs file for rings !\n");
439 amdgpu_ring_lockup_update(ring
);
444 * amdgpu_ring_fini - tear down the driver ring struct.
446 * @adev: amdgpu_device pointer
447 * @ring: amdgpu_ring structure holding ring information
449 * Tear down the driver information for the selected ring (all asics).
451 void amdgpu_ring_fini(struct amdgpu_ring
*ring
)
454 struct amdgpu_bo
*ring_obj
;
456 if (ring
->ring_lock
== NULL
)
459 mutex_lock(ring
->ring_lock
);
460 ring_obj
= ring
->ring_obj
;
463 ring
->ring_obj
= NULL
;
464 mutex_unlock(ring
->ring_lock
);
466 amdgpu_wb_free(ring
->adev
, ring
->fence_offs
);
467 amdgpu_wb_free(ring
->adev
, ring
->rptr_offs
);
468 amdgpu_wb_free(ring
->adev
, ring
->wptr_offs
);
469 amdgpu_wb_free(ring
->adev
, ring
->next_rptr_offs
);
472 r
= amdgpu_bo_reserve(ring_obj
, false);
473 if (likely(r
== 0)) {
474 amdgpu_bo_kunmap(ring_obj
);
475 amdgpu_bo_unpin(ring_obj
);
476 amdgpu_bo_unreserve(ring_obj
);
478 amdgpu_bo_unref(&ring_obj
);
485 #if defined(CONFIG_DEBUG_FS)
487 static int amdgpu_debugfs_ring_info(struct seq_file
*m
, void *data
)
489 struct drm_info_node
*node
= (struct drm_info_node
*) m
->private;
490 struct drm_device
*dev
= node
->minor
->dev
;
491 struct amdgpu_device
*adev
= dev
->dev_private
;
492 int roffset
= *(int*)node
->info_ent
->data
;
493 struct amdgpu_ring
*ring
= (void *)(((uint8_t*)adev
) + roffset
);
495 uint32_t rptr
, wptr
, rptr_next
;
496 unsigned count
, i
, j
;
498 amdgpu_ring_free_size(ring
);
499 count
= (ring
->ring_size
/ 4) - ring
->ring_free_dw
;
501 wptr
= amdgpu_ring_get_wptr(ring
);
502 seq_printf(m
, "wptr: 0x%08x [%5d]\n",
505 rptr
= amdgpu_ring_get_rptr(ring
);
506 seq_printf(m
, "rptr: 0x%08x [%5d]\n",
511 seq_printf(m
, "driver's copy of the wptr: 0x%08x [%5d]\n",
512 ring
->wptr
, ring
->wptr
);
513 seq_printf(m
, "last semaphore signal addr : 0x%016llx\n",
514 ring
->last_semaphore_signal_addr
);
515 seq_printf(m
, "last semaphore wait addr : 0x%016llx\n",
516 ring
->last_semaphore_wait_addr
);
517 seq_printf(m
, "%u free dwords in ring\n", ring
->ring_free_dw
);
518 seq_printf(m
, "%u dwords in ring\n", count
);
523 /* print 8 dw before current rptr as often it's the last executed
524 * packet that is the root issue
526 i
= (rptr
+ ring
->ptr_mask
+ 1 - 32) & ring
->ptr_mask
;
527 for (j
= 0; j
<= (count
+ 32); j
++) {
528 seq_printf(m
, "r[%5d]=0x%08x", i
, ring
->ring
[i
]);
534 i
= (i
+ 1) & ring
->ptr_mask
;
539 /* TODO: clean this up !*/
540 static int amdgpu_gfx_index
= offsetof(struct amdgpu_device
, gfx
.gfx_ring
[0]);
541 static int cayman_cp1_index
= offsetof(struct amdgpu_device
, gfx
.compute_ring
[0]);
542 static int cayman_cp2_index
= offsetof(struct amdgpu_device
, gfx
.compute_ring
[1]);
543 static int amdgpu_dma1_index
= offsetof(struct amdgpu_device
, sdma
[0].ring
);
544 static int amdgpu_dma2_index
= offsetof(struct amdgpu_device
, sdma
[1].ring
);
545 static int r600_uvd_index
= offsetof(struct amdgpu_device
, uvd
.ring
);
546 static int si_vce1_index
= offsetof(struct amdgpu_device
, vce
.ring
[0]);
547 static int si_vce2_index
= offsetof(struct amdgpu_device
, vce
.ring
[1]);
549 static struct drm_info_list amdgpu_debugfs_ring_info_list
[] = {
550 {"amdgpu_ring_gfx", amdgpu_debugfs_ring_info
, 0, &amdgpu_gfx_index
},
551 {"amdgpu_ring_cp1", amdgpu_debugfs_ring_info
, 0, &cayman_cp1_index
},
552 {"amdgpu_ring_cp2", amdgpu_debugfs_ring_info
, 0, &cayman_cp2_index
},
553 {"amdgpu_ring_dma1", amdgpu_debugfs_ring_info
, 0, &amdgpu_dma1_index
},
554 {"amdgpu_ring_dma2", amdgpu_debugfs_ring_info
, 0, &amdgpu_dma2_index
},
555 {"amdgpu_ring_uvd", amdgpu_debugfs_ring_info
, 0, &r600_uvd_index
},
556 {"amdgpu_ring_vce1", amdgpu_debugfs_ring_info
, 0, &si_vce1_index
},
557 {"amdgpu_ring_vce2", amdgpu_debugfs_ring_info
, 0, &si_vce2_index
},
562 static int amdgpu_debugfs_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
)
564 #if defined(CONFIG_DEBUG_FS)
566 for (i
= 0; i
< ARRAY_SIZE(amdgpu_debugfs_ring_info_list
); ++i
) {
567 struct drm_info_list
*info
= &amdgpu_debugfs_ring_info_list
[i
];
568 int roffset
= *(int*)amdgpu_debugfs_ring_info_list
[i
].data
;
569 struct amdgpu_ring
*other
= (void *)(((uint8_t*)adev
) + roffset
);
575 r
= amdgpu_debugfs_add_files(adev
, info
, 1);