2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/types.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/printk.h>
28 #include <linux/sched.h>
29 #include "kfd_kernel_queue.h"
31 #include "kfd_device_queue_manager.h"
32 #include "kfd_pm4_headers.h"
33 #include "kfd_pm4_opcodes.h"
35 #define PM4_COUNT_ZERO (((1 << 15) - 1) << 16)
37 static bool initialize(struct kernel_queue
*kq
, struct kfd_dev
*dev
,
38 enum kfd_queue_type type
, unsigned int queue_size
)
40 struct queue_properties prop
;
42 union PM4_MES_TYPE_3_HEADER nop
;
45 BUG_ON(type
!= KFD_QUEUE_TYPE_DIQ
&& type
!= KFD_QUEUE_TYPE_HIQ
);
47 pr_debug("kfd: In func %s initializing queue type %d size %d\n",
48 __func__
, KFD_QUEUE_TYPE_HIQ
, queue_size
);
51 nop
.type
= PM4_TYPE_3
;
52 nop
.u32all
|= PM4_COUNT_ZERO
;
55 kq
->nop_packet
= nop
.u32all
;
57 case KFD_QUEUE_TYPE_DIQ
:
58 case KFD_QUEUE_TYPE_HIQ
:
59 kq
->mqd
= dev
->dqm
->ops
.get_mqd_manager(dev
->dqm
,
70 prop
.doorbell_ptr
= kfd_get_kernel_doorbell(dev
, &prop
.doorbell_off
);
72 if (prop
.doorbell_ptr
== NULL
)
73 goto err_get_kernel_doorbell
;
75 retval
= kfd_gtt_sa_allocate(dev
, queue_size
, &kq
->pq
);
77 goto err_pq_allocate_vidmem
;
79 kq
->pq_kernel_addr
= kq
->pq
->cpu_ptr
;
80 kq
->pq_gpu_addr
= kq
->pq
->gpu_addr
;
82 retval
= kq
->ops_asic_specific
.initialize(kq
, dev
, type
, queue_size
);
84 goto err_eop_allocate_vidmem
;
86 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->rptr_kernel
),
90 goto err_rptr_allocate_vidmem
;
92 kq
->rptr_kernel
= kq
->rptr_mem
->cpu_ptr
;
93 kq
->rptr_gpu_addr
= kq
->rptr_mem
->gpu_addr
;
95 retval
= kfd_gtt_sa_allocate(dev
, sizeof(*kq
->wptr_kernel
),
99 goto err_wptr_allocate_vidmem
;
101 kq
->wptr_kernel
= kq
->wptr_mem
->cpu_ptr
;
102 kq
->wptr_gpu_addr
= kq
->wptr_mem
->gpu_addr
;
104 memset(kq
->pq_kernel_addr
, 0, queue_size
);
105 memset(kq
->rptr_kernel
, 0, sizeof(*kq
->rptr_kernel
));
106 memset(kq
->wptr_kernel
, 0, sizeof(*kq
->wptr_kernel
));
108 prop
.queue_size
= queue_size
;
109 prop
.is_interop
= false;
111 prop
.queue_percent
= 100;
114 prop
.queue_address
= kq
->pq_gpu_addr
;
115 prop
.read_ptr
= (uint32_t *) kq
->rptr_gpu_addr
;
116 prop
.write_ptr
= (uint32_t *) kq
->wptr_gpu_addr
;
117 prop
.eop_ring_buffer_address
= kq
->eop_gpu_addr
;
118 prop
.eop_ring_buffer_size
= PAGE_SIZE
;
120 if (init_queue(&kq
->queue
, prop
) != 0)
123 kq
->queue
->device
= dev
;
124 kq
->queue
->process
= kfd_get_process(current
);
126 retval
= kq
->mqd
->init_mqd(kq
->mqd
, &kq
->queue
->mqd
,
127 &kq
->queue
->mqd_mem_obj
,
128 &kq
->queue
->gart_mqd_addr
,
129 &kq
->queue
->properties
);
133 /* assign HIQ to HQD */
134 if (type
== KFD_QUEUE_TYPE_HIQ
) {
135 pr_debug("assigning hiq to hqd\n");
136 kq
->queue
->pipe
= KFD_CIK_HIQ_PIPE
;
137 kq
->queue
->queue
= KFD_CIK_HIQ_QUEUE
;
138 kq
->mqd
->load_mqd(kq
->mqd
, kq
->queue
->mqd
, kq
->queue
->pipe
,
139 kq
->queue
->queue
, NULL
);
141 /* allocate fence for DIQ */
143 retval
= kfd_gtt_sa_allocate(dev
, sizeof(uint32_t),
147 goto err_alloc_fence
;
149 kq
->fence_kernel_address
= kq
->fence_mem_obj
->cpu_ptr
;
150 kq
->fence_gpu_addr
= kq
->fence_mem_obj
->gpu_addr
;
153 print_queue(kq
->queue
);
158 uninit_queue(kq
->queue
);
160 kfd_gtt_sa_free(dev
, kq
->wptr_mem
);
161 err_wptr_allocate_vidmem
:
162 kfd_gtt_sa_free(dev
, kq
->rptr_mem
);
163 err_rptr_allocate_vidmem
:
164 kfd_gtt_sa_free(dev
, kq
->eop_mem
);
165 err_eop_allocate_vidmem
:
166 kfd_gtt_sa_free(dev
, kq
->pq
);
167 err_pq_allocate_vidmem
:
168 pr_err("kfd: error init pq\n");
169 kfd_release_kernel_doorbell(dev
, prop
.doorbell_ptr
);
170 err_get_kernel_doorbell
:
171 pr_err("kfd: error init doorbell");
176 static void uninitialize(struct kernel_queue
*kq
)
180 if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_HIQ
)
181 kq
->mqd
->destroy_mqd(kq
->mqd
,
184 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS
,
187 else if (kq
->queue
->properties
.type
== KFD_QUEUE_TYPE_DIQ
)
188 kfd_gtt_sa_free(kq
->dev
, kq
->fence_mem_obj
);
190 kfd_gtt_sa_free(kq
->dev
, kq
->rptr_mem
);
191 kfd_gtt_sa_free(kq
->dev
, kq
->wptr_mem
);
192 kq
->ops_asic_specific
.uninitialize(kq
);
193 kfd_gtt_sa_free(kq
->dev
, kq
->pq
);
194 kfd_release_kernel_doorbell(kq
->dev
,
195 kq
->queue
->properties
.doorbell_ptr
);
196 uninit_queue(kq
->queue
);
199 static int acquire_packet_buffer(struct kernel_queue
*kq
,
200 size_t packet_size_in_dwords
, unsigned int **buffer_ptr
)
202 size_t available_size
;
203 size_t queue_size_dwords
;
205 unsigned int *queue_address
;
207 BUG_ON(!kq
|| !buffer_ptr
);
209 rptr
= *kq
->rptr_kernel
;
210 wptr
= *kq
->wptr_kernel
;
211 queue_address
= (unsigned int *)kq
->pq_kernel_addr
;
212 queue_size_dwords
= kq
->queue
->properties
.queue_size
/ sizeof(uint32_t);
214 pr_debug("kfd: In func %s\nrptr: %d\nwptr: %d\nqueue_address 0x%p\n",
215 __func__
, rptr
, wptr
, queue_address
);
217 available_size
= (rptr
- 1 - wptr
+ queue_size_dwords
) %
220 if (packet_size_in_dwords
>= queue_size_dwords
||
221 packet_size_in_dwords
>= available_size
) {
223 * make sure calling functions know
224 * acquire_packet_buffer() failed
230 if (wptr
+ packet_size_in_dwords
>= queue_size_dwords
) {
232 queue_address
[wptr
] = kq
->nop_packet
;
233 wptr
= (wptr
+ 1) % queue_size_dwords
;
237 *buffer_ptr
= &queue_address
[wptr
];
238 kq
->pending_wptr
= wptr
+ packet_size_in_dwords
;
243 static void submit_packet(struct kernel_queue
*kq
)
252 for (i
= *kq
->wptr_kernel
; i
< kq
->pending_wptr
; i
++) {
253 pr_debug("0x%2X ", kq
->pq_kernel_addr
[i
]);
260 *kq
->wptr_kernel
= kq
->pending_wptr
;
261 write_kernel_doorbell(kq
->queue
->properties
.doorbell_ptr
,
265 static void rollback_packet(struct kernel_queue
*kq
)
268 kq
->pending_wptr
= *kq
->queue
->properties
.write_ptr
;
271 struct kernel_queue
*kernel_queue_init(struct kfd_dev
*dev
,
272 enum kfd_queue_type type
)
274 struct kernel_queue
*kq
;
278 kq
= kzalloc(sizeof(struct kernel_queue
), GFP_KERNEL
);
282 kq
->ops
.initialize
= initialize
;
283 kq
->ops
.uninitialize
= uninitialize
;
284 kq
->ops
.acquire_packet_buffer
= acquire_packet_buffer
;
285 kq
->ops
.submit_packet
= submit_packet
;
286 kq
->ops
.rollback_packet
= rollback_packet
;
288 switch (dev
->device_info
->asic_family
) {
290 kernel_queue_init_vi(&kq
->ops_asic_specific
);
294 kernel_queue_init_cik(&kq
->ops_asic_specific
);
298 if (kq
->ops
.initialize(kq
, dev
, type
, KFD_KERNEL_QUEUE_SIZE
) == false) {
299 pr_err("kfd: failed to init kernel queue\n");
306 void kernel_queue_uninit(struct kernel_queue
*kq
)
310 kq
->ops
.uninitialize(kq
);
314 static __attribute__((unused
)) void test_kq(struct kfd_dev
*dev
)
316 struct kernel_queue
*kq
;
322 pr_err("kfd: starting kernel queue test\n");
324 kq
= kernel_queue_init(dev
, KFD_QUEUE_TYPE_HIQ
);
327 retval
= kq
->ops
.acquire_packet_buffer(kq
, 5, &buffer
);
329 for (i
= 0; i
< 5; i
++)
330 buffer
[i
] = kq
->nop_packet
;
331 kq
->ops
.submit_packet(kq
);
333 pr_err("kfd: ending kernel queue test\n");