2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
29 #include <linux/seq_file.h>
30 #include <linux/slab.h>
32 #include <drm/amdgpu_drm.h>
38 * Most engines on the GPU are fed via ring buffers. Ring
39 * buffers are areas of GPU accessible memory that the host
40 * writes commands into and the GPU reads commands out of.
41 * There is a rptr (read pointer) that determines where the
42 * GPU is currently reading, and a wptr (write pointer)
43 * which determines where the host has written. When the
44 * pointers are equal, the ring is idle. When the host
45 * writes commands to the ring buffer, it increments the
46 * wptr. The GPU then starts fetching commands and executes
47 * them until the pointers are equal again.
49 static int amdgpu_debugfs_ring_init(struct amdgpu_device
*adev
,
50 struct amdgpu_ring
*ring
);
53 * amdgpu_ring_alloc - allocate space on the ring buffer
55 * @adev: amdgpu_device pointer
56 * @ring: amdgpu_ring structure holding ring information
57 * @ndw: number of dwords to allocate in the ring buffer
59 * Allocate @ndw dwords in the ring buffer (all asics).
60 * Returns 0 on success, error on failure.
62 int amdgpu_ring_alloc(struct amdgpu_ring
*ring
, unsigned ndw
)
64 /* Align requested size with padding so unlock_commit can
66 ndw
= (ndw
+ ring
->align_mask
) & ~ring
->align_mask
;
68 /* Make sure we aren't trying to allocate more space
69 * than the maximum for one submission
71 if (WARN_ON_ONCE(ndw
> ring
->max_dw
))
75 ring
->wptr_old
= ring
->wptr
;
79 /** amdgpu_ring_insert_nop - insert NOP packets
81 * @ring: amdgpu_ring structure holding ring information
82 * @count: the number of NOP packets to insert
84 * This is the generic insert_nop function for rings except SDMA
86 void amdgpu_ring_insert_nop(struct amdgpu_ring
*ring
, uint32_t count
)
90 for (i
= 0; i
< count
; i
++)
91 amdgpu_ring_write(ring
, ring
->nop
);
94 /** amdgpu_ring_generic_pad_ib - pad IB with NOP packets
96 * @ring: amdgpu_ring structure holding ring information
97 * @ib: IB to add NOP packets to
99 * This is the generic pad_ib function for rings except SDMA
101 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring
*ring
, struct amdgpu_ib
*ib
)
103 while (ib
->length_dw
& ring
->align_mask
)
104 ib
->ptr
[ib
->length_dw
++] = ring
->nop
;
108 * amdgpu_ring_commit - tell the GPU to execute the new
109 * commands on the ring buffer
111 * @adev: amdgpu_device pointer
112 * @ring: amdgpu_ring structure holding ring information
114 * Update the wptr (write pointer) to tell the GPU to
115 * execute new commands on the ring buffer (all asics).
117 void amdgpu_ring_commit(struct amdgpu_ring
*ring
)
121 /* We pad to match fetch size */
122 count
= ring
->align_mask
+ 1 - (ring
->wptr
& ring
->align_mask
);
123 count
%= ring
->align_mask
+ 1;
124 ring
->funcs
->insert_nop(ring
, count
);
127 amdgpu_ring_set_wptr(ring
);
131 * amdgpu_ring_undo - reset the wptr
133 * @ring: amdgpu_ring structure holding ring information
135 * Reset the driver's copy of the wptr (all asics).
137 void amdgpu_ring_undo(struct amdgpu_ring
*ring
)
139 ring
->wptr
= ring
->wptr_old
;
143 * amdgpu_ring_backup - Back up the content of a ring
145 * @ring: the ring we want to back up
147 * Saves all unprocessed commits from a ring, returns the number of dwords saved.
149 unsigned amdgpu_ring_backup(struct amdgpu_ring
*ring
,
152 unsigned size
, ptr
, i
;
156 if (ring
->ring_obj
== NULL
)
159 /* it doesn't make sense to save anything if all fences are signaled */
160 if (!amdgpu_fence_count_emitted(ring
))
163 ptr
= le32_to_cpu(*ring
->next_rptr_cpu_addr
);
165 size
= ring
->wptr
+ (ring
->ring_size
/ 4);
167 size
&= ring
->ptr_mask
;
171 /* and then save the content of the ring */
172 *data
= kmalloc_array(size
, sizeof(uint32_t), GFP_KERNEL
);
175 for (i
= 0; i
< size
; ++i
) {
176 (*data
)[i
] = ring
->ring
[ptr
++];
177 ptr
&= ring
->ptr_mask
;
184 * amdgpu_ring_restore - append saved commands to the ring again
186 * @ring: ring to append commands to
187 * @size: number of dwords we want to write
188 * @data: saved commands
190 * Allocates space on the ring and restore the previously saved commands.
192 int amdgpu_ring_restore(struct amdgpu_ring
*ring
,
193 unsigned size
, uint32_t *data
)
200 /* restore the saved ring content */
201 r
= amdgpu_ring_alloc(ring
, size
);
205 for (i
= 0; i
< size
; ++i
) {
206 amdgpu_ring_write(ring
, data
[i
]);
209 amdgpu_ring_commit(ring
);
215 * amdgpu_ring_init - init driver ring struct.
217 * @adev: amdgpu_device pointer
218 * @ring: amdgpu_ring structure holding ring information
219 * @max_ndw: maximum number of dw for ring alloc
220 * @nop: nop packet for this ring
222 * Initialize the driver information for the selected ring (all asics).
223 * Returns 0 on success, error on failure.
225 int amdgpu_ring_init(struct amdgpu_device
*adev
, struct amdgpu_ring
*ring
,
226 unsigned max_dw
, u32 nop
, u32 align_mask
,
227 struct amdgpu_irq_src
*irq_src
, unsigned irq_type
,
228 enum amdgpu_ring_type ring_type
)
232 if (ring
->adev
== NULL
) {
233 if (adev
->num_rings
>= AMDGPU_MAX_RINGS
)
237 ring
->idx
= adev
->num_rings
++;
238 adev
->rings
[ring
->idx
] = ring
;
239 r
= amdgpu_fence_driver_init_ring(ring
,
240 amdgpu_sched_hw_submission
);
245 r
= amdgpu_wb_get(adev
, &ring
->rptr_offs
);
247 dev_err(adev
->dev
, "(%d) ring rptr_offs wb alloc failed\n", r
);
251 r
= amdgpu_wb_get(adev
, &ring
->wptr_offs
);
253 dev_err(adev
->dev
, "(%d) ring wptr_offs wb alloc failed\n", r
);
257 r
= amdgpu_wb_get(adev
, &ring
->fence_offs
);
259 dev_err(adev
->dev
, "(%d) ring fence_offs wb alloc failed\n", r
);
263 r
= amdgpu_wb_get(adev
, &ring
->next_rptr_offs
);
265 dev_err(adev
->dev
, "(%d) ring next_rptr wb alloc failed\n", r
);
268 ring
->next_rptr_gpu_addr
= adev
->wb
.gpu_addr
+ ring
->next_rptr_offs
* 4;
269 ring
->next_rptr_cpu_addr
= &adev
->wb
.wb
[ring
->next_rptr_offs
];
271 r
= amdgpu_wb_get(adev
, &ring
->cond_exe_offs
);
273 dev_err(adev
->dev
, "(%d) ring cond_exec_polling wb alloc failed\n", r
);
276 ring
->cond_exe_gpu_addr
= adev
->wb
.gpu_addr
+ (ring
->cond_exe_offs
* 4);
277 ring
->cond_exe_cpu_addr
= &adev
->wb
.wb
[ring
->cond_exe_offs
];
279 spin_lock_init(&ring
->fence_lock
);
280 r
= amdgpu_fence_driver_start_ring(ring
, irq_src
, irq_type
);
282 dev_err(adev
->dev
, "failed initializing fences (%d).\n", r
);
286 ring
->ring_size
= roundup_pow_of_two(max_dw
* 4 *
287 amdgpu_sched_hw_submission
);
288 ring
->align_mask
= align_mask
;
290 ring
->type
= ring_type
;
292 /* Allocate ring buffer */
293 if (ring
->ring_obj
== NULL
) {
294 r
= amdgpu_bo_create(adev
, ring
->ring_size
, PAGE_SIZE
, true,
295 AMDGPU_GEM_DOMAIN_GTT
, 0,
296 NULL
, NULL
, &ring
->ring_obj
);
298 dev_err(adev
->dev
, "(%d) ring create failed\n", r
);
301 r
= amdgpu_bo_reserve(ring
->ring_obj
, false);
302 if (unlikely(r
!= 0))
304 r
= amdgpu_bo_pin(ring
->ring_obj
, AMDGPU_GEM_DOMAIN_GTT
,
307 amdgpu_bo_unreserve(ring
->ring_obj
);
308 dev_err(adev
->dev
, "(%d) ring pin failed\n", r
);
311 r
= amdgpu_bo_kmap(ring
->ring_obj
,
312 (void **)&ring
->ring
);
313 amdgpu_bo_unreserve(ring
->ring_obj
);
315 dev_err(adev
->dev
, "(%d) ring map failed\n", r
);
319 ring
->ptr_mask
= (ring
->ring_size
/ 4) - 1;
320 ring
->max_dw
= max_dw
;
322 if (amdgpu_debugfs_ring_init(adev
, ring
)) {
323 DRM_ERROR("Failed to register debugfs file for rings !\n");
329 * amdgpu_ring_fini - tear down the driver ring struct.
331 * @adev: amdgpu_device pointer
332 * @ring: amdgpu_ring structure holding ring information
334 * Tear down the driver information for the selected ring (all asics).
336 void amdgpu_ring_fini(struct amdgpu_ring
*ring
)
339 struct amdgpu_bo
*ring_obj
;
341 ring_obj
= ring
->ring_obj
;
344 ring
->ring_obj
= NULL
;
346 amdgpu_wb_free(ring
->adev
, ring
->cond_exe_offs
);
347 amdgpu_wb_free(ring
->adev
, ring
->fence_offs
);
348 amdgpu_wb_free(ring
->adev
, ring
->rptr_offs
);
349 amdgpu_wb_free(ring
->adev
, ring
->wptr_offs
);
350 amdgpu_wb_free(ring
->adev
, ring
->next_rptr_offs
);
353 r
= amdgpu_bo_reserve(ring_obj
, false);
354 if (likely(r
== 0)) {
355 amdgpu_bo_kunmap(ring_obj
);
356 amdgpu_bo_unpin(ring_obj
);
357 amdgpu_bo_unreserve(ring_obj
);
359 amdgpu_bo_unref(&ring_obj
);
366 #if defined(CONFIG_DEBUG_FS)
368 static int amdgpu_debugfs_ring_info(struct seq_file
*m
, void *data
)
370 struct drm_info_node
*node
= (struct drm_info_node
*) m
->private;
371 struct drm_device
*dev
= node
->minor
->dev
;
372 struct amdgpu_device
*adev
= dev
->dev_private
;
373 int roffset
= (unsigned long)node
->info_ent
->data
;
374 struct amdgpu_ring
*ring
= (void *)(((uint8_t*)adev
) + roffset
);
375 uint32_t rptr
, wptr
, rptr_next
;
378 wptr
= amdgpu_ring_get_wptr(ring
);
379 seq_printf(m
, "wptr: 0x%08x [%5d]\n", wptr
, wptr
);
381 rptr
= amdgpu_ring_get_rptr(ring
);
382 rptr_next
= le32_to_cpu(*ring
->next_rptr_cpu_addr
);
384 seq_printf(m
, "rptr: 0x%08x [%5d]\n", rptr
, rptr
);
386 seq_printf(m
, "driver's copy of the wptr: 0x%08x [%5d]\n",
387 ring
->wptr
, ring
->wptr
);
392 /* print 8 dw before current rptr as often it's the last executed
393 * packet that is the root issue
395 i
= (rptr
+ ring
->ptr_mask
+ 1 - 32) & ring
->ptr_mask
;
397 seq_printf(m
, "r[%5d]=0x%08x", i
, ring
->ring
[i
]);
403 i
= (i
+ 1) & ring
->ptr_mask
;
406 seq_printf(m
, "r[%5d]=0x%08x", i
, ring
->ring
[i
]);
412 i
= (i
+ 1) & ring
->ptr_mask
;
417 static struct drm_info_list amdgpu_debugfs_ring_info_list
[AMDGPU_MAX_RINGS
];
418 static char amdgpu_debugfs_ring_names
[AMDGPU_MAX_RINGS
][32];
422 static int amdgpu_debugfs_ring_init(struct amdgpu_device
*adev
,
423 struct amdgpu_ring
*ring
)
425 #if defined(CONFIG_DEBUG_FS)
426 unsigned offset
= (uint8_t*)ring
- (uint8_t*)adev
;
428 struct drm_info_list
*info
;
431 for (i
= 0; i
< ARRAY_SIZE(amdgpu_debugfs_ring_info_list
); ++i
) {
432 info
= &amdgpu_debugfs_ring_info_list
[i
];
437 if (i
== ARRAY_SIZE(amdgpu_debugfs_ring_info_list
))
440 name
= &amdgpu_debugfs_ring_names
[i
][0];
441 sprintf(name
, "amdgpu_ring_%s", ring
->name
);
443 info
->show
= amdgpu_debugfs_ring_info
;
444 info
->driver_features
= 0;
445 info
->data
= (void*)(uintptr_t)offset
;
447 return amdgpu_debugfs_add_files(adev
, info
, 1);