2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/types.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/printk.h>
28 #include <linux/sched.h>
29 #include "kfd_kernel_queue.h"
31 #include "kfd_device_queue_manager.h"
32 #include "kfd_pm4_headers.h"
33 #include "kfd_pm4_opcodes.h"
35 #define PM4_COUNT_ZERO (((1 << 15) - 1) << 16)
37 static bool initialize(struct kernel_queue
*kq
, struct kfd_dev
*dev
,
38 enum kfd_queue_type type
, unsigned int queue_size
)
40 struct queue_properties prop
;
42 union PM4_MES_TYPE_3_HEADER nop
;
45 BUG_ON(type
!= KFD_QUEUE_TYPE_DIQ
&& type
!= KFD_QUEUE_TYPE_HIQ
);
47 pr_debug("amdkfd: In func %s initializing queue type %d size %d\n",
48 __func__
, KFD_QUEUE_TYPE_HIQ
, queue_size
);
51 nop
.type
= PM4_TYPE_3
;
52 nop
.u32all
|= PM4_COUNT_ZERO
;
55 kq
->nop_packet
= nop
.u32all
;
57 case KFD_QUEUE_TYPE_DIQ
:
58 case KFD_QUEUE_TYPE_HIQ
:
59 kq
->mqd
= dev
->dqm
->ops
.get_mqd_manager(dev
->dqm
,
70 prop
.doorbell_ptr
= kfd_get_kernel_doorbell(dev
, &prop
.doorbell_off
);
72 if (prop
.doorbell_ptr
== NULL
) {
73 pr_err("amdkfd: error init doorbell");
74 goto err_get_kernel_doorbell
;
77 retval
= kfd_gtt_sa_allocate(dev
, queue_size
, &kq
->pq
);
79 pr_err("amdkfd: error init pq queues size (%d)\n", queue_size
);
80 goto err_pq_allocate_vidmem
;
83 kq
->pq_kernel_addr
= kq
->pq
->cpu_ptr
;
84 kq
->pq_gpu_addr
= kq
->pq
->gpu_addr
;
86 retval
= kq
->ops_asic_specific
.initialize(kq
, dev
, type
, queue_size
);
88 goto err_eop_allocate_vidmem
;
90 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->rptr_kernel
),
94 goto err_rptr_allocate_vidmem
;
96 kq
->rptr_kernel
= kq
->rptr_mem
->cpu_ptr
;
97 kq
->rptr_gpu_addr
= kq
->rptr_mem
->gpu_addr
;
99 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->wptr_kernel
),
103 goto err_wptr_allocate_vidmem
;
105 kq
->wptr_kernel
= kq
->wptr_mem
->cpu_ptr
;
106 kq
->wptr_gpu_addr
= kq
->wptr_mem
->gpu_addr
;
108 memset(kq
->pq_kernel_addr
, 0, queue_size
);
109 memset(kq
->rptr_kernel
, 0, sizeof(*kq
->rptr_kernel
));
110 memset(kq
->wptr_kernel
, 0, sizeof(*kq
->wptr_kernel
));
112 prop
.queue_size
= queue_size
;
113 prop
.is_interop
= false;
115 prop
.queue_percent
= 100;
118 prop
.queue_address
= kq
->pq_gpu_addr
;
119 prop
.read_ptr
= (uint32_t *) kq
->rptr_gpu_addr
;
120 prop
.write_ptr
= (uint32_t *) kq
->wptr_gpu_addr
;
121 prop
.eop_ring_buffer_address
= kq
->eop_gpu_addr
;
122 prop
.eop_ring_buffer_size
= PAGE_SIZE
;
124 if (init_queue(&kq
->queue
, prop
) != 0)
127 kq
->queue
->device
= dev
;
128 kq
->queue
->process
= kfd_get_process(current
);
130 retval
= kq
->mqd
->init_mqd(kq
->mqd
, &kq
->queue
->mqd
,
131 &kq
->queue
->mqd_mem_obj
,
132 &kq
->queue
->gart_mqd_addr
,
133 &kq
->queue
->properties
);
137 /* assign HIQ to HQD */
138 if (type
== KFD_QUEUE_TYPE_HIQ
) {
139 pr_debug("assigning hiq to hqd\n");
140 kq
->queue
->pipe
= KFD_CIK_HIQ_PIPE
;
141 kq
->queue
->queue
= KFD_CIK_HIQ_QUEUE
;
142 kq
->mqd
->load_mqd(kq
->mqd
, kq
->queue
->mqd
, kq
->queue
->pipe
,
143 kq
->queue
->queue
, NULL
);
145 /* allocate fence for DIQ */
147 retval
= kfd_gtt_sa_allocate(dev
, sizeof(uint32_t),
151 goto err_alloc_fence
;
153 kq
->fence_kernel_address
= kq
->fence_mem_obj
->cpu_ptr
;
154 kq
->fence_gpu_addr
= kq
->fence_mem_obj
->gpu_addr
;
157 print_queue(kq
->queue
);
162 uninit_queue(kq
->queue
);
164 kfd_gtt_sa_free(dev
, kq
->wptr_mem
);
165 err_wptr_allocate_vidmem
:
166 kfd_gtt_sa_free(dev
, kq
->rptr_mem
);
167 err_rptr_allocate_vidmem
:
168 kfd_gtt_sa_free(dev
, kq
->eop_mem
);
169 err_eop_allocate_vidmem
:
170 kfd_gtt_sa_free(dev
, kq
->pq
);
171 err_pq_allocate_vidmem
:
172 kfd_release_kernel_doorbell(dev
, prop
.doorbell_ptr
);
173 err_get_kernel_doorbell
:
178 static void uninitialize(struct kernel_queue
*kq
)
182 if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_HIQ
)
183 kq
->mqd
->destroy_mqd(kq
->mqd
,
186 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS
,
189 else if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_DIQ
)
190 kfd_gtt_sa_free(kq
->dev
, kq
->fence_mem_obj
);
192 kq
->mqd
->uninit_mqd(kq
->mqd
, kq
->queue
->mqd
, kq
->queue
->mqd_mem_obj
);
194 kfd_gtt_sa_free(kq
->dev
, kq
->rptr_mem
);
195 kfd_gtt_sa_free(kq
->dev
, kq
->wptr_mem
);
196 kq
->ops_asic_specific
.uninitialize(kq
);
197 kfd_gtt_sa_free(kq
->dev
, kq
->pq
);
198 kfd_release_kernel_doorbell(kq
->dev
,
199 kq
->queue
->properties
.doorbell_ptr
);
200 uninit_queue(kq
->queue
);
203 static int acquire_packet_buffer(struct kernel_queue
*kq
,
204 size_t packet_size_in_dwords
, unsigned int **buffer_ptr
)
206 size_t available_size
;
207 size_t queue_size_dwords
;
209 unsigned int *queue_address
;
211 BUG_ON(!kq
|| !buffer_ptr
);
213 rptr
= *kq
->rptr_kernel
;
214 wptr
= *kq
->wptr_kernel
;
215 queue_address
= (unsigned int *)kq
->pq_kernel_addr
;
216 queue_size_dwords
= kq
->queue
->properties
.queue_size
/ sizeof(uint32_t);
218 pr_debug("rptr: %d\n", rptr
);
219 pr_debug("wptr: %d\n", wptr
);
220 pr_debug("queue_address 0x%p\n", queue_address
);
222 available_size
= (rptr
- 1 - wptr
+ queue_size_dwords
) %
225 if (packet_size_in_dwords
>= queue_size_dwords
||
226 packet_size_in_dwords
>= available_size
) {
228 * make sure calling functions know
229 * acquire_packet_buffer() failed
235 if (wptr
+ packet_size_in_dwords
>= queue_size_dwords
) {
237 queue_address
[wptr
] = kq
->nop_packet
;
238 wptr
= (wptr
+ 1) % queue_size_dwords
;
242 *buffer_ptr
= &queue_address
[wptr
];
243 kq
->pending_wptr
= wptr
+ packet_size_in_dwords
;
248 static void submit_packet(struct kernel_queue
*kq
)
257 for (i
= *kq
->wptr_kernel
; i
< kq
->pending_wptr
; i
++) {
258 pr_debug("0x%2X ", kq
->pq_kernel_addr
[i
]);
265 *kq
->wptr_kernel
= kq
->pending_wptr
;
266 write_kernel_doorbell(kq
->queue
->properties
.doorbell_ptr
,
270 static void rollback_packet(struct kernel_queue
*kq
)
273 kq
->pending_wptr
= *kq
->queue
->properties
.write_ptr
;
276 struct kernel_queue
*kernel_queue_init(struct kfd_dev
*dev
,
277 enum kfd_queue_type type
)
279 struct kernel_queue
*kq
;
283 kq
= kzalloc(sizeof(struct kernel_queue
), GFP_KERNEL
);
287 kq
->ops
.initialize
= initialize
;
288 kq
->ops
.uninitialize
= uninitialize
;
289 kq
->ops
.acquire_packet_buffer
= acquire_packet_buffer
;
290 kq
->ops
.submit_packet
= submit_packet
;
291 kq
->ops
.rollback_packet
= rollback_packet
;
293 switch (dev
->device_info
->asic_family
) {
295 kernel_queue_init_vi(&kq
->ops_asic_specific
);
299 kernel_queue_init_cik(&kq
->ops_asic_specific
);
303 if (kq
->ops
.initialize(kq
, dev
, type
, KFD_KERNEL_QUEUE_SIZE
) == false) {
304 pr_err("amdkfd: failed to init kernel queue\n");
311 void kernel_queue_uninit(struct kernel_queue
*kq
)
315 kq
->ops
.uninitialize(kq
);
319 static __attribute__((unused
)) void test_kq(struct kfd_dev
*dev
)
321 struct kernel_queue
*kq
;
327 pr_err("amdkfd: starting kernel queue test\n");
329 kq
= kernel_queue_init(dev
, KFD_QUEUE_TYPE_HIQ
);
332 retval
= kq
->ops
.acquire_packet_buffer(kq
, 5, &buffer
);
334 for (i
= 0; i
< 5; i
++)
335 buffer
[i
] = kq
->nop_packet
;
336 kq
->ops
.submit_packet(kq
);
338 pr_err("amdkfd: ending kernel queue test\n");