2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/types.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/printk.h>
28 #include <linux/sched.h>
29 #include "kfd_kernel_queue.h"
31 #include "kfd_device_queue_manager.h"
32 #include "kfd_pm4_headers.h"
33 #include "kfd_pm4_opcodes.h"
35 #define PM4_COUNT_ZERO (((1 << 15) - 1) << 16)
37 static bool initialize(struct kernel_queue
*kq
, struct kfd_dev
*dev
,
38 enum kfd_queue_type type
, unsigned int queue_size
)
40 struct queue_properties prop
;
42 union PM4_MES_TYPE_3_HEADER nop
;
45 BUG_ON(type
!= KFD_QUEUE_TYPE_DIQ
&& type
!= KFD_QUEUE_TYPE_HIQ
);
47 pr_debug("kfd: In func %s initializing queue type %d size %d\n",
48 __func__
, KFD_QUEUE_TYPE_HIQ
, queue_size
);
51 nop
.type
= PM4_TYPE_3
;
52 nop
.u32all
|= PM4_COUNT_ZERO
;
55 kq
->nop_packet
= nop
.u32all
;
57 case KFD_QUEUE_TYPE_DIQ
:
58 case KFD_QUEUE_TYPE_HIQ
:
59 kq
->mqd
= dev
->dqm
->ops
.get_mqd_manager(dev
->dqm
,
70 prop
.doorbell_ptr
= kfd_get_kernel_doorbell(dev
, &prop
.doorbell_off
);
72 if (prop
.doorbell_ptr
== NULL
)
73 goto err_get_kernel_doorbell
;
75 retval
= kfd_gtt_sa_allocate(dev
, queue_size
, &kq
->pq
);
78 goto err_pq_allocate_vidmem
;
80 kq
->pq_kernel_addr
= kq
->pq
->cpu_ptr
;
81 kq
->pq_gpu_addr
= kq
->pq
->gpu_addr
;
83 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->rptr_kernel
),
87 goto err_rptr_allocate_vidmem
;
89 kq
->rptr_kernel
= kq
->rptr_mem
->cpu_ptr
;
90 kq
->rptr_gpu_addr
= kq
->rptr_mem
->gpu_addr
;
92 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->wptr_kernel
),
96 goto err_wptr_allocate_vidmem
;
98 kq
->wptr_kernel
= kq
->wptr_mem
->cpu_ptr
;
99 kq
->wptr_gpu_addr
= kq
->wptr_mem
->gpu_addr
;
101 memset(kq
->pq_kernel_addr
, 0, queue_size
);
102 memset(kq
->rptr_kernel
, 0, sizeof(*kq
->rptr_kernel
));
103 memset(kq
->wptr_kernel
, 0, sizeof(*kq
->wptr_kernel
));
105 prop
.queue_size
= queue_size
;
106 prop
.is_interop
= false;
108 prop
.queue_percent
= 100;
111 prop
.queue_address
= kq
->pq_gpu_addr
;
112 prop
.read_ptr
= (uint32_t *) kq
->rptr_gpu_addr
;
113 prop
.write_ptr
= (uint32_t *) kq
->wptr_gpu_addr
;
115 if (init_queue(&kq
->queue
, prop
) != 0)
118 kq
->queue
->device
= dev
;
119 kq
->queue
->process
= kfd_get_process(current
);
121 retval
= kq
->mqd
->init_mqd(kq
->mqd
, &kq
->queue
->mqd
,
122 &kq
->queue
->mqd_mem_obj
,
123 &kq
->queue
->gart_mqd_addr
,
124 &kq
->queue
->properties
);
128 /* assign HIQ to HQD */
129 if (type
== KFD_QUEUE_TYPE_HIQ
) {
130 pr_debug("assigning hiq to hqd\n");
131 kq
->queue
->pipe
= KFD_CIK_HIQ_PIPE
;
132 kq
->queue
->queue
= KFD_CIK_HIQ_QUEUE
;
133 kq
->mqd
->load_mqd(kq
->mqd
, kq
->queue
->mqd
, kq
->queue
->pipe
,
134 kq
->queue
->queue
, NULL
);
136 /* allocate fence for DIQ */
138 retval
= kfd_gtt_sa_allocate(dev
, sizeof(uint32_t),
142 goto err_alloc_fence
;
144 kq
->fence_kernel_address
= kq
->fence_mem_obj
->cpu_ptr
;
145 kq
->fence_gpu_addr
= kq
->fence_mem_obj
->gpu_addr
;
148 print_queue(kq
->queue
);
153 uninit_queue(kq
->queue
);
155 kfd_gtt_sa_free(dev
, kq
->wptr_mem
);
156 err_wptr_allocate_vidmem
:
157 kfd_gtt_sa_free(dev
, kq
->rptr_mem
);
158 err_rptr_allocate_vidmem
:
159 kfd_gtt_sa_free(dev
, kq
->pq
);
160 err_pq_allocate_vidmem
:
161 pr_err("kfd: error init pq\n");
162 kfd_release_kernel_doorbell(dev
, prop
.doorbell_ptr
);
163 err_get_kernel_doorbell
:
164 pr_err("kfd: error init doorbell");
169 static void uninitialize(struct kernel_queue
*kq
)
173 if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_HIQ
)
174 kq
->mqd
->destroy_mqd(kq
->mqd
,
177 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS
,
180 else if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_DIQ
)
181 kfd_gtt_sa_free(kq
->dev
, kq
->fence_mem_obj
);
183 kfd_gtt_sa_free(kq
->dev
, kq
->rptr_mem
);
184 kfd_gtt_sa_free(kq
->dev
, kq
->wptr_mem
);
185 kfd_gtt_sa_free(kq
->dev
, kq
->pq
);
186 kfd_release_kernel_doorbell(kq
->dev
,
187 kq
->queue
->properties
.doorbell_ptr
);
188 uninit_queue(kq
->queue
);
191 static int acquire_packet_buffer(struct kernel_queue
*kq
,
192 size_t packet_size_in_dwords
, unsigned int **buffer_ptr
)
194 size_t available_size
;
195 size_t queue_size_dwords
;
197 unsigned int *queue_address
;
199 BUG_ON(!kq
|| !buffer_ptr
);
201 rptr
= *kq
->rptr_kernel
;
202 wptr
= *kq
->wptr_kernel
;
203 queue_address
= (unsigned int *)kq
->pq_kernel_addr
;
204 queue_size_dwords
= kq
->queue
->properties
.queue_size
/ sizeof(uint32_t);
206 pr_debug("kfd: In func %s\nrptr: %d\nwptr: %d\nqueue_address 0x%p\n",
207 __func__
, rptr
, wptr
, queue_address
);
209 available_size
= (rptr
- 1 - wptr
+ queue_size_dwords
) %
212 if (packet_size_in_dwords
>= queue_size_dwords
||
213 packet_size_in_dwords
>= available_size
) {
215 * make sure calling functions know
216 * acquire_packet_buffer() failed
222 if (wptr
+ packet_size_in_dwords
>= queue_size_dwords
) {
224 queue_address
[wptr
] = kq
->nop_packet
;
225 wptr
= (wptr
+ 1) % queue_size_dwords
;
229 *buffer_ptr
= &queue_address
[wptr
];
230 kq
->pending_wptr
= wptr
+ packet_size_in_dwords
;
235 static void submit_packet(struct kernel_queue
*kq
)
244 for (i
= *kq
->wptr_kernel
; i
< kq
->pending_wptr
; i
++) {
245 pr_debug("0x%2X ", kq
->pq_kernel_addr
[i
]);
252 *kq
->wptr_kernel
= kq
->pending_wptr
;
253 write_kernel_doorbell(kq
->queue
->properties
.doorbell_ptr
,
257 static int sync_with_hw(struct kernel_queue
*kq
, unsigned long timeout_ms
)
259 unsigned long org_timeout_ms
;
263 org_timeout_ms
= timeout_ms
;
264 timeout_ms
+= jiffies
* 1000 / HZ
;
265 while (*kq
->wptr_kernel
!= *kq
->rptr_kernel
) {
266 if (time_after(jiffies
* 1000 / HZ
, timeout_ms
)) {
267 pr_err("kfd: kernel_queue %s timeout expired %lu\n",
268 __func__
, org_timeout_ms
);
269 pr_err("kfd: wptr: %d rptr: %d\n",
270 *kq
->wptr_kernel
, *kq
->rptr_kernel
);
279 static void rollback_packet(struct kernel_queue
*kq
)
282 kq
->pending_wptr
= *kq
->queue
->properties
.write_ptr
;
285 struct kernel_queue
*kernel_queue_init(struct kfd_dev
*dev
,
286 enum kfd_queue_type type
)
288 struct kernel_queue
*kq
;
292 kq
= kzalloc(sizeof(struct kernel_queue
), GFP_KERNEL
);
296 kq
->initialize
= initialize
;
297 kq
->uninitialize
= uninitialize
;
298 kq
->acquire_packet_buffer
= acquire_packet_buffer
;
299 kq
->submit_packet
= submit_packet
;
300 kq
->sync_with_hw
= sync_with_hw
;
301 kq
->rollback_packet
= rollback_packet
;
303 if (kq
->initialize(kq
, dev
, type
, KFD_KERNEL_QUEUE_SIZE
) == false) {
304 pr_err("kfd: failed to init kernel queue\n");
311 void kernel_queue_uninit(struct kernel_queue
*kq
)
315 kq
->uninitialize(kq
);
319 static __attribute__((unused
)) void test_kq(struct kfd_dev
*dev
)
321 struct kernel_queue
*kq
;
327 pr_debug("kfd: starting kernel queue test\n");
329 kq
= kernel_queue_init(dev
, KFD_QUEUE_TYPE_HIQ
);
332 retval
= kq
->acquire_packet_buffer(kq
, 5, &buffer
);
334 for (i
= 0; i
< 5; i
++)
335 buffer
[i
] = kq
->nop_packet
;
336 kq
->submit_packet(kq
);
337 kq
->sync_with_hw(kq
, 1000);
339 pr_debug("kfd: ending kernel queue test\n");