Commit | Line | Data |
---|---|---|
4b8f589b BG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
24 | #include <linux/printk.h> | |
25 | #include <linux/slab.h> | |
26 | #include "kfd_priv.h" | |
27 | #include "kfd_mqd_manager.h" | |
28 | #include "cik_regs.h" | |
29 | #include "cik_structs.h" | |
30 | ||
4b8f589b BG |
31 | static inline struct cik_mqd *get_mqd(void *mqd) |
32 | { | |
33 | return (struct cik_mqd *)mqd; | |
34 | } | |
35 | ||
36 | static int init_mqd(struct mqd_manager *mm, void **mqd, | |
37 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
38 | struct queue_properties *q) | |
39 | { | |
40 | uint64_t addr; | |
41 | struct cik_mqd *m; | |
42 | int retval; | |
43 | ||
44 | BUG_ON(!mm || !q || !mqd); | |
45 | ||
46 | pr_debug("kfd: In func %s\n", __func__); | |
47 | ||
48 | retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), | |
49 | mqd_mem_obj); | |
50 | ||
51 | if (retval != 0) | |
52 | return -ENOMEM; | |
53 | ||
54 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
55 | addr = (*mqd_mem_obj)->gpu_addr; | |
56 | ||
57 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
58 | ||
59 | m->header = 0xC0310800; | |
60 | m->compute_pipelinestat_enable = 1; | |
61 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
62 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
63 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
64 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
65 | ||
66 | /* | |
67 | * Make sure to use the last queue state saved on mqd when the cp | |
68 | * reassigns the queue, so when queue is switched on/off (e.g over | |
69 | * subscription or quantum timeout) the context will be consistent | |
70 | */ | |
71 | m->cp_hqd_persistent_state = | |
72 | DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; | |
73 | ||
74 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
75 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
76 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
77 | ||
78 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; | |
79 | /* Although WinKFD writes this, I suspect it should not be necessary */ | |
80 | m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; | |
81 | ||
82 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
83 | QUANTUM_DURATION(10); | |
84 | ||
85 | /* | |
86 | * Pipe Priority | |
87 | * Identifies the pipe relative priority when this queue is connected | |
88 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
89 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
90 | * 0 = CS_LOW (typically below GFX) | |
91 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
92 | * 2 = CS_HIGH (typically above HP3D) | |
93 | */ | |
94 | m->cp_hqd_pipe_priority = 1; | |
95 | m->cp_hqd_queue_priority = 15; | |
96 | ||
d752f95e JC |
97 | if (q->format == KFD_QUEUE_FORMAT_AQL) |
98 | m->cp_hqd_iq_rptr = AQL_ENABLE; | |
99 | ||
4b8f589b BG |
100 | *mqd = m; |
101 | if (gart_addr != NULL) | |
102 | *gart_addr = addr; | |
103 | retval = mm->update_mqd(mm, m, q); | |
104 | ||
105 | return retval; | |
106 | } | |
107 | ||
108 | static int init_mqd_sdma(struct mqd_manager *mm, void **mqd, | |
109 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
110 | struct queue_properties *q) | |
111 | { | |
112 | int retval; | |
113 | struct cik_sdma_rlc_registers *m; | |
114 | ||
115 | BUG_ON(!mm || !mqd || !mqd_mem_obj); | |
116 | ||
117 | retval = kfd_gtt_sa_allocate(mm->dev, | |
118 | sizeof(struct cik_sdma_rlc_registers), | |
119 | mqd_mem_obj); | |
120 | ||
121 | if (retval != 0) | |
122 | return -ENOMEM; | |
123 | ||
124 | m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr; | |
125 | ||
126 | memset(m, 0, sizeof(struct cik_sdma_rlc_registers)); | |
127 | ||
128 | *mqd = m; | |
129 | if (gart_addr != NULL) | |
130 | *gart_addr = (*mqd_mem_obj)->gpu_addr; | |
131 | ||
132 | retval = mm->update_mqd(mm, m, q); | |
133 | ||
134 | return retval; | |
135 | } | |
136 | ||
137 | static void uninit_mqd(struct mqd_manager *mm, void *mqd, | |
138 | struct kfd_mem_obj *mqd_mem_obj) | |
139 | { | |
140 | BUG_ON(!mm || !mqd); | |
141 | kfd_gtt_sa_free(mm->dev, mqd_mem_obj); | |
142 | } | |
143 | ||
144 | static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
145 | struct kfd_mem_obj *mqd_mem_obj) | |
146 | { | |
147 | BUG_ON(!mm || !mqd); | |
148 | kfd_gtt_sa_free(mm->dev, mqd_mem_obj); | |
149 | } | |
150 | ||
151 | static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, | |
152 | uint32_t queue_id, uint32_t __user *wptr) | |
153 | { | |
cea405b1 XZ |
154 | return mm->dev->kfd2kgd->hqd_load |
155 | (mm->dev->kgd, mqd, pipe_id, queue_id, wptr); | |
4b8f589b BG |
156 | } |
157 | ||
158 | static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
159 | uint32_t pipe_id, uint32_t queue_id, | |
160 | uint32_t __user *wptr) | |
161 | { | |
cea405b1 | 162 | return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd); |
4b8f589b BG |
163 | } |
164 | ||
165 | static int update_mqd(struct mqd_manager *mm, void *mqd, | |
166 | struct queue_properties *q) | |
167 | { | |
168 | struct cik_mqd *m; | |
169 | ||
170 | BUG_ON(!mm || !q || !mqd); | |
171 | ||
172 | pr_debug("kfd: In func %s\n", __func__); | |
173 | ||
174 | m = get_mqd(mqd); | |
175 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
176 | DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; | |
177 | ||
178 | /* | |
179 | * Calculating queue size which is log base 2 of actual queue size -1 | |
180 | * dwords and another -1 for ffs | |
181 | */ | |
182 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
183 | - 1 - 1; | |
184 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
185 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
186 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
187 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
188 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
189 | DOORBELL_OFFSET(q->doorbell_off); | |
190 | ||
191 | m->cp_hqd_vmid = q->vmid; | |
192 | ||
193 | if (q->format == KFD_QUEUE_FORMAT_AQL) { | |
4b8f589b BG |
194 | m->cp_hqd_pq_control |= NO_UPDATE_RPTR; |
195 | } | |
196 | ||
197 | m->cp_hqd_active = 0; | |
198 | q->is_active = false; | |
199 | if (q->queue_size > 0 && | |
200 | q->queue_address != 0 && | |
201 | q->queue_percent > 0) { | |
202 | m->cp_hqd_active = 1; | |
203 | q->is_active = true; | |
204 | } | |
205 | ||
206 | return 0; | |
207 | } | |
208 | ||
209 | static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
210 | struct queue_properties *q) | |
211 | { | |
212 | struct cik_sdma_rlc_registers *m; | |
213 | ||
214 | BUG_ON(!mm || !mqd || !q); | |
215 | ||
216 | m = get_sdma_mqd(mqd); | |
217 | m->sdma_rlc_rb_cntl = | |
218 | SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) | | |
219 | SDMA_RB_VMID(q->vmid) | | |
220 | SDMA_RPTR_WRITEBACK_ENABLE | | |
221 | SDMA_RPTR_WRITEBACK_TIMER(6); | |
222 | ||
223 | m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); | |
224 | m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); | |
225 | m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
226 | m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
227 | m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE; | |
228 | m->sdma_rlc_virtual_addr = q->sdma_vm_addr; | |
229 | ||
230 | m->sdma_engine_id = q->sdma_engine_id; | |
231 | m->sdma_queue_id = q->sdma_queue_id; | |
232 | ||
233 | q->is_active = false; | |
234 | if (q->queue_size > 0 && | |
235 | q->queue_address != 0 && | |
236 | q->queue_percent > 0) { | |
237 | m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE; | |
238 | q->is_active = true; | |
239 | } | |
240 | ||
241 | return 0; | |
242 | } | |
243 | ||
244 | static int destroy_mqd(struct mqd_manager *mm, void *mqd, | |
245 | enum kfd_preempt_type type, | |
246 | unsigned int timeout, uint32_t pipe_id, | |
247 | uint32_t queue_id) | |
248 | { | |
cea405b1 | 249 | return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, |
4b8f589b BG |
250 | pipe_id, queue_id); |
251 | } | |
252 | ||
253 | /* | |
254 | * preempt type here is ignored because there is only one way | |
255 | * to preempt sdma queue | |
256 | */ | |
257 | static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
258 | enum kfd_preempt_type type, | |
259 | unsigned int timeout, uint32_t pipe_id, | |
260 | uint32_t queue_id) | |
261 | { | |
cea405b1 | 262 | return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout); |
4b8f589b BG |
263 | } |
264 | ||
265 | static bool is_occupied(struct mqd_manager *mm, void *mqd, | |
266 | uint64_t queue_address, uint32_t pipe_id, | |
267 | uint32_t queue_id) | |
268 | { | |
269 | ||
cea405b1 | 270 | return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address, |
4b8f589b BG |
271 | pipe_id, queue_id); |
272 | ||
273 | } | |
274 | ||
275 | static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, | |
276 | uint64_t queue_address, uint32_t pipe_id, | |
277 | uint32_t queue_id) | |
278 | { | |
cea405b1 | 279 | return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd); |
4b8f589b BG |
280 | } |
281 | ||
282 | /* | |
283 | * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. | |
284 | * The HIQ queue in Kaveri is using the same MQD structure as all the user mode | |
285 | * queues but with different initial values. | |
286 | */ | |
287 | ||
288 | static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, | |
289 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
290 | struct queue_properties *q) | |
291 | { | |
292 | uint64_t addr; | |
293 | struct cik_mqd *m; | |
294 | int retval; | |
295 | ||
296 | BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); | |
297 | ||
298 | pr_debug("kfd: In func %s\n", __func__); | |
299 | ||
300 | retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), | |
301 | mqd_mem_obj); | |
302 | ||
303 | if (retval != 0) | |
304 | return -ENOMEM; | |
305 | ||
306 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
307 | addr = (*mqd_mem_obj)->gpu_addr; | |
308 | ||
309 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
310 | ||
311 | m->header = 0xC0310800; | |
312 | m->compute_pipelinestat_enable = 1; | |
313 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
314 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
315 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
316 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
317 | ||
318 | m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | | |
319 | PRELOAD_REQ; | |
320 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
321 | QUANTUM_DURATION(10); | |
322 | ||
323 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
324 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
325 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
326 | ||
327 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; | |
328 | ||
329 | /* | |
330 | * Pipe Priority | |
331 | * Identifies the pipe relative priority when this queue is connected | |
332 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
333 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
334 | * 0 = CS_LOW (typically below GFX) | |
335 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
336 | * 2 = CS_HIGH (typically above HP3D) | |
337 | */ | |
338 | m->cp_hqd_pipe_priority = 1; | |
339 | m->cp_hqd_queue_priority = 15; | |
340 | ||
341 | *mqd = m; | |
342 | if (gart_addr) | |
343 | *gart_addr = addr; | |
344 | retval = mm->update_mqd(mm, m, q); | |
345 | ||
346 | return retval; | |
347 | } | |
348 | ||
349 | static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, | |
350 | struct queue_properties *q) | |
351 | { | |
352 | struct cik_mqd *m; | |
353 | ||
354 | BUG_ON(!mm || !q || !mqd); | |
355 | ||
356 | pr_debug("kfd: In func %s\n", __func__); | |
357 | ||
358 | m = get_mqd(mqd); | |
359 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
360 | DEFAULT_MIN_AVAIL_SIZE | | |
361 | PRIV_STATE | | |
362 | KMD_QUEUE; | |
363 | ||
364 | /* | |
365 | * Calculating queue size which is log base 2 of actual queue | |
366 | * size -1 dwords | |
367 | */ | |
368 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
369 | - 1 - 1; | |
370 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
371 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
372 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
373 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
374 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
375 | DOORBELL_OFFSET(q->doorbell_off); | |
376 | ||
377 | m->cp_hqd_vmid = q->vmid; | |
378 | ||
379 | m->cp_hqd_active = 0; | |
380 | q->is_active = false; | |
381 | if (q->queue_size > 0 && | |
382 | q->queue_address != 0 && | |
383 | q->queue_percent > 0) { | |
384 | m->cp_hqd_active = 1; | |
385 | q->is_active = true; | |
386 | } | |
387 | ||
388 | return 0; | |
389 | } | |
390 | ||
391 | struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) | |
392 | { | |
393 | struct cik_sdma_rlc_registers *m; | |
394 | ||
395 | BUG_ON(!mqd); | |
396 | ||
397 | m = (struct cik_sdma_rlc_registers *)mqd; | |
398 | ||
399 | return m; | |
400 | } | |
401 | ||
402 | struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, | |
403 | struct kfd_dev *dev) | |
404 | { | |
405 | struct mqd_manager *mqd; | |
406 | ||
407 | BUG_ON(!dev); | |
408 | BUG_ON(type >= KFD_MQD_TYPE_MAX); | |
409 | ||
410 | pr_debug("kfd: In func %s\n", __func__); | |
411 | ||
412 | mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); | |
413 | if (!mqd) | |
414 | return NULL; | |
415 | ||
416 | mqd->dev = dev; | |
417 | ||
418 | switch (type) { | |
419 | case KFD_MQD_TYPE_CP: | |
420 | case KFD_MQD_TYPE_COMPUTE: | |
421 | mqd->init_mqd = init_mqd; | |
422 | mqd->uninit_mqd = uninit_mqd; | |
423 | mqd->load_mqd = load_mqd; | |
424 | mqd->update_mqd = update_mqd; | |
425 | mqd->destroy_mqd = destroy_mqd; | |
426 | mqd->is_occupied = is_occupied; | |
427 | break; | |
428 | case KFD_MQD_TYPE_HIQ: | |
429 | mqd->init_mqd = init_mqd_hiq; | |
430 | mqd->uninit_mqd = uninit_mqd; | |
431 | mqd->load_mqd = load_mqd; | |
432 | mqd->update_mqd = update_mqd_hiq; | |
433 | mqd->destroy_mqd = destroy_mqd; | |
434 | mqd->is_occupied = is_occupied; | |
435 | break; | |
436 | case KFD_MQD_TYPE_SDMA: | |
437 | mqd->init_mqd = init_mqd_sdma; | |
438 | mqd->uninit_mqd = uninit_mqd_sdma; | |
439 | mqd->load_mqd = load_mqd_sdma; | |
440 | mqd->update_mqd = update_mqd_sdma; | |
441 | mqd->destroy_mqd = destroy_mqd_sdma; | |
442 | mqd->is_occupied = is_occupied_sdma; | |
443 | break; | |
444 | default: | |
445 | kfree(mqd); | |
446 | return NULL; | |
447 | } | |
448 | ||
449 | return mqd; | |
450 | } | |
451 |