2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
29 #include <linux/seq_file.h>
30 #include <linux/slab.h>
32 #include <drm/amdgpu_drm.h>
38 * Most engines on the GPU are fed via ring buffers. Ring
39 * buffers are areas of GPU accessible memory that the host
40 * writes commands into and the GPU reads commands out of.
41 * There is a rptr (read pointer) that determines where the
42 * GPU is currently reading, and a wptr (write pointer)
43 * which determines where the host has written. When the
44 * pointers are equal, the ring is idle. When the host
45 * writes commands to the ring buffer, it increments the
46 * wptr. The GPU then starts fetching commands and executes
47 * them until the pointers are equal again.
49 static int amdgpu_debugfs_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
);
52 * amdgpu_ring_alloc - allocate space on the ring buffer
54 * @adev: amdgpu_device pointer
55 * @ring: amdgpu_ring structure holding ring information
56 * @ndw: number of dwords to allocate in the ring buffer
58 * Allocate @ndw dwords in the ring buffer (all asics).
59 * Returns 0 on success, error on failure.
61 int amdgpu_ring_alloc(struct amdgpu_ring
*ring
, unsigned ndw
)
63 /* Align requested size with padding so unlock_commit can
65 ndw
= (ndw
+ ring
->align_mask
) & ~ring
->align_mask
;
67 /* Make sure we aren't trying to allocate more space
68 * than the maximum for one submission
70 if (WARN_ON_ONCE(ndw
> ring
->max_dw
))
74 ring
->wptr_old
= ring
->wptr
;
78 /** amdgpu_ring_insert_nop - insert NOP packets
80 * @ring: amdgpu_ring structure holding ring information
81 * @count: the number of NOP packets to insert
83 * This is the generic insert_nop function for rings except SDMA
85 void amdgpu_ring_insert_nop(struct amdgpu_ring
*ring
, uint32_t count
)
89 for (i
= 0; i
< count
; i
++)
90 amdgpu_ring_write(ring
, ring
->nop
);
93 /** amdgpu_ring_generic_pad_ib - pad IB with NOP packets
95 * @ring: amdgpu_ring structure holding ring information
96 * @ib: IB to add NOP packets to
98 * This is the generic pad_ib function for rings except SDMA
100 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring
*ring
, struct amdgpu_ib
*ib
)
102 while (ib
->length_dw
& ring
->align_mask
)
103 ib
->ptr
[ib
->length_dw
++] = ring
->nop
;
107 * amdgpu_ring_commit - tell the GPU to execute the new
108 * commands on the ring buffer
110 * @adev: amdgpu_device pointer
111 * @ring: amdgpu_ring structure holding ring information
113 * Update the wptr (write pointer) to tell the GPU to
114 * execute new commands on the ring buffer (all asics).
116 void amdgpu_ring_commit(struct amdgpu_ring
*ring
)
120 /* We pad to match fetch size */
121 count
= ring
->align_mask
+ 1 - (ring
->wptr
& ring
->align_mask
);
122 count
%= ring
->align_mask
+ 1;
123 ring
->funcs
->insert_nop(ring
, count
);
126 amdgpu_ring_set_wptr(ring
);
130 * amdgpu_ring_undo - reset the wptr
132 * @ring: amdgpu_ring structure holding ring information
134 * Reset the driver's copy of the wptr (all asics).
136 void amdgpu_ring_undo(struct amdgpu_ring
*ring
)
138 ring
->wptr
= ring
->wptr_old
;
142 * amdgpu_ring_backup - Back up the content of a ring
144 * @ring: the ring we want to back up
146 * Saves all unprocessed commits from a ring, returns the number of dwords saved.
148 unsigned amdgpu_ring_backup(struct amdgpu_ring
*ring
,
151 unsigned size
, ptr
, i
;
155 if (ring
->ring_obj
== NULL
)
158 /* it doesn't make sense to save anything if all fences are signaled */
159 if (!amdgpu_fence_count_emitted(ring
))
162 ptr
= le32_to_cpu(*ring
->next_rptr_cpu_addr
);
164 size
= ring
->wptr
+ (ring
->ring_size
/ 4);
166 size
&= ring
->ptr_mask
;
170 /* and then save the content of the ring */
171 *data
= kmalloc_array(size
, sizeof(uint32_t), GFP_KERNEL
);
174 for (i
= 0; i
< size
; ++i
) {
175 (*data
)[i
] = ring
->ring
[ptr
++];
176 ptr
&= ring
->ptr_mask
;
183 * amdgpu_ring_restore - append saved commands to the ring again
185 * @ring: ring to append commands to
186 * @size: number of dwords we want to write
187 * @data: saved commands
189 * Allocates space on the ring and restore the previously saved commands.
191 int amdgpu_ring_restore(struct amdgpu_ring
*ring
,
192 unsigned size
, uint32_t *data
)
199 /* restore the saved ring content */
200 r
= amdgpu_ring_alloc(ring
, size
);
204 for (i
= 0; i
< size
; ++i
) {
205 amdgpu_ring_write(ring
, data
[i
]);
208 amdgpu_ring_commit(ring
);
214 * amdgpu_ring_init - init driver ring struct.
216 * @adev: amdgpu_device pointer
217 * @ring: amdgpu_ring structure holding ring information
218 * @max_ndw: maximum number of dw for ring alloc
219 * @nop: nop packet for this ring
221 * Initialize the driver information for the selected ring (all asics).
222 * Returns 0 on success, error on failure.
224 int amdgpu_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
,
225 unsigned max_dw
, u32 nop
, u32 align_mask
,
226 struct amdgpu_irq_src
*irq_src
, unsigned irq_type
,
227 enum amdgpu_ring_type ring_type
)
231 if (ring
->adev
== NULL
) {
232 if (adev
->num_rings
>= AMDGPU_MAX_RINGS
)
236 ring
->idx
= adev
->num_rings
++;
237 adev
->rings
[ring
->idx
] = ring
;
238 r
= amdgpu_fence_driver_init_ring(ring
,
239 amdgpu_sched_hw_submission
);
244 r
= amdgpu_wb_get(adev
, &ring
->rptr_offs
);
246 dev_err(adev
->dev
, "(%d) ring rptr_offs wb alloc failed\n", r
);
250 r
= amdgpu_wb_get(adev
, &ring
->wptr_offs
);
252 dev_err(adev
->dev
, "(%d) ring wptr_offs wb alloc failed\n", r
);
256 r
= amdgpu_wb_get(adev
, &ring
->fence_offs
);
258 dev_err(adev
->dev
, "(%d) ring fence_offs wb alloc failed\n", r
);
262 r
= amdgpu_wb_get(adev
, &ring
->next_rptr_offs
);
264 dev_err(adev
->dev
, "(%d) ring next_rptr wb alloc failed\n", r
);
267 ring
->next_rptr_gpu_addr
= adev
->wb
.gpu_addr
+ (ring
->next_rptr_offs
* 4);
268 ring
->next_rptr_cpu_addr
= &adev
->wb
.wb
[ring
->next_rptr_offs
];
270 r
= amdgpu_wb_get(adev
, &ring
->cond_exe_offs
);
272 dev_err(adev
->dev
, "(%d) ring cond_exec_polling wb alloc failed\n", r
);
275 ring
->cond_exe_gpu_addr
= adev
->wb
.gpu_addr
+ (ring
->cond_exe_offs
* 4);
276 ring
->cond_exe_cpu_addr
= &adev
->wb
.wb
[ring
->cond_exe_offs
];
278 spin_lock_init(&ring
->fence_lock
);
279 r
= amdgpu_fence_driver_start_ring(ring
, irq_src
, irq_type
);
281 dev_err(adev
->dev
, "failed initializing fences (%d).\n", r
);
285 ring
->ring_size
= roundup_pow_of_two(max_dw
* 4 *
286 amdgpu_sched_hw_submission
);
287 ring
->align_mask
= align_mask
;
289 ring
->type
= ring_type
;
291 /* Allocate ring buffer */
292 if (ring
->ring_obj
== NULL
) {
293 r
= amdgpu_bo_create(adev
, ring
->ring_size
, PAGE_SIZE
, true,
294 AMDGPU_GEM_DOMAIN_GTT
, 0,
295 NULL
, NULL
, &ring
->ring_obj
);
297 dev_err(adev
->dev
, "(%d) ring create failed\n", r
);
300 r
= amdgpu_bo_reserve(ring
->ring_obj
, false);
301 if (unlikely(r
!= 0))
303 r
= amdgpu_bo_pin(ring
->ring_obj
, AMDGPU_GEM_DOMAIN_GTT
,
306 amdgpu_bo_unreserve(ring
->ring_obj
);
307 dev_err(adev
->dev
, "(%d) ring pin failed\n", r
);
310 r
= amdgpu_bo_kmap(ring
->ring_obj
,
311 (void **)&ring
->ring
);
312 amdgpu_bo_unreserve(ring
->ring_obj
);
314 dev_err(adev
->dev
, "(%d) ring map failed\n", r
);
318 ring
->ptr_mask
= (ring
->ring_size
/ 4) - 1;
319 ring
->max_dw
= max_dw
;
321 if (amdgpu_debugfs_ring_init(adev
, ring
)) {
322 DRM_ERROR("Failed to register debugfs file for rings !\n");
328 * amdgpu_ring_fini - tear down the driver ring struct.
330 * @adev: amdgpu_device pointer
331 * @ring: amdgpu_ring structure holding ring information
333 * Tear down the driver information for the selected ring (all asics).
335 void amdgpu_ring_fini(struct amdgpu_ring
*ring
)
338 struct amdgpu_bo
*ring_obj
;
340 ring_obj
= ring
->ring_obj
;
343 ring
->ring_obj
= NULL
;
345 amdgpu_wb_free(ring
->adev
, ring
->fence_offs
);
346 amdgpu_wb_free(ring
->adev
, ring
->rptr_offs
);
347 amdgpu_wb_free(ring
->adev
, ring
->wptr_offs
);
348 amdgpu_wb_free(ring
->adev
, ring
->next_rptr_offs
);
351 r
= amdgpu_bo_reserve(ring_obj
, false);
352 if (likely(r
== 0)) {
353 amdgpu_bo_kunmap(ring_obj
);
354 amdgpu_bo_unpin(ring_obj
);
355 amdgpu_bo_unreserve(ring_obj
);
357 amdgpu_bo_unref(&ring_obj
);
364 #if defined(CONFIG_DEBUG_FS)
366 static int amdgpu_debugfs_ring_info(struct seq_file
*m
, void *data
)
368 struct drm_info_node
*node
= (struct drm_info_node
*) m
->private;
369 struct drm_device
*dev
= node
->minor
->dev
;
370 struct amdgpu_device
*adev
= dev
->dev_private
;
371 int roffset
= *(int*)node
->info_ent
->data
;
372 struct amdgpu_ring
*ring
= (void *)(((uint8_t*)adev
) + roffset
);
374 uint32_t rptr
, wptr
, rptr_next
;
377 wptr
= amdgpu_ring_get_wptr(ring
);
378 seq_printf(m
, "wptr: 0x%08x [%5d]\n", wptr
, wptr
);
380 rptr
= amdgpu_ring_get_rptr(ring
);
381 rptr_next
= le32_to_cpu(*ring
->next_rptr_cpu_addr
);
383 seq_printf(m
, "rptr: 0x%08x [%5d]\n", rptr
, rptr
);
385 seq_printf(m
, "driver's copy of the wptr: 0x%08x [%5d]\n",
386 ring
->wptr
, ring
->wptr
);
391 /* print 8 dw before current rptr as often it's the last executed
392 * packet that is the root issue
394 i
= (rptr
+ ring
->ptr_mask
+ 1 - 32) & ring
->ptr_mask
;
396 seq_printf(m
, "r[%5d]=0x%08x", i
, ring
->ring
[i
]);
402 i
= (i
+ 1) & ring
->ptr_mask
;
405 seq_printf(m
, "r[%5d]=0x%08x", i
, ring
->ring
[i
]);
411 i
= (i
+ 1) & ring
->ptr_mask
;
416 /* TODO: clean this up !*/
417 static int amdgpu_gfx_index
= offsetof(struct amdgpu_device
, gfx
.gfx_ring
[0]);
418 static int cayman_cp1_index
= offsetof(struct amdgpu_device
, gfx
.compute_ring
[0]);
419 static int cayman_cp2_index
= offsetof(struct amdgpu_device
, gfx
.compute_ring
[1]);
420 static int amdgpu_dma1_index
= offsetof(struct amdgpu_device
, sdma
.instance
[0].ring
);
421 static int amdgpu_dma2_index
= offsetof(struct amdgpu_device
, sdma
.instance
[1].ring
);
422 static int r600_uvd_index
= offsetof(struct amdgpu_device
, uvd
.ring
);
423 static int si_vce1_index
= offsetof(struct amdgpu_device
, vce
.ring
[0]);
424 static int si_vce2_index
= offsetof(struct amdgpu_device
, vce
.ring
[1]);
426 static const struct drm_info_list amdgpu_debugfs_ring_info_list
[] = {
427 {"amdgpu_ring_gfx", amdgpu_debugfs_ring_info
, 0, &amdgpu_gfx_index
},
428 {"amdgpu_ring_cp1", amdgpu_debugfs_ring_info
, 0, &cayman_cp1_index
},
429 {"amdgpu_ring_cp2", amdgpu_debugfs_ring_info
, 0, &cayman_cp2_index
},
430 {"amdgpu_ring_dma1", amdgpu_debugfs_ring_info
, 0, &amdgpu_dma1_index
},
431 {"amdgpu_ring_dma2", amdgpu_debugfs_ring_info
, 0, &amdgpu_dma2_index
},
432 {"amdgpu_ring_uvd", amdgpu_debugfs_ring_info
, 0, &r600_uvd_index
},
433 {"amdgpu_ring_vce1", amdgpu_debugfs_ring_info
, 0, &si_vce1_index
},
434 {"amdgpu_ring_vce2", amdgpu_debugfs_ring_info
, 0, &si_vce2_index
},
439 static int amdgpu_debugfs_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
)
441 #if defined(CONFIG_DEBUG_FS)
443 for (i
= 0; i
< ARRAY_SIZE(amdgpu_debugfs_ring_info_list
); ++i
) {
444 const struct drm_info_list
*info
= &amdgpu_debugfs_ring_info_list
[i
];
445 int roffset
= *(int*)amdgpu_debugfs_ring_info_list
[i
].data
;
446 struct amdgpu_ring
*other
= (void *)(((uint8_t*)adev
) + roffset
);
452 r
= amdgpu_debugfs_add_files(adev
, info
, 1);