Commit | Line | Data |
---|---|---|
4b8f589b BG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
24 | #include <linux/printk.h> | |
25 | #include <linux/slab.h> | |
26 | #include "kfd_priv.h" | |
27 | #include "kfd_mqd_manager.h" | |
28 | #include "cik_regs.h" | |
29 | #include "cik_structs.h" | |
3d30b28b | 30 | #include "oss/oss_2_4_sh_mask.h" |
4b8f589b | 31 | |
4b8f589b BG |
32 | static inline struct cik_mqd *get_mqd(void *mqd) |
33 | { | |
34 | return (struct cik_mqd *)mqd; | |
35 | } | |
36 | ||
37 | static int init_mqd(struct mqd_manager *mm, void **mqd, | |
38 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
39 | struct queue_properties *q) | |
40 | { | |
41 | uint64_t addr; | |
42 | struct cik_mqd *m; | |
43 | int retval; | |
44 | ||
45 | BUG_ON(!mm || !q || !mqd); | |
46 | ||
47 | pr_debug("kfd: In func %s\n", __func__); | |
48 | ||
49 | retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), | |
50 | mqd_mem_obj); | |
51 | ||
52 | if (retval != 0) | |
53 | return -ENOMEM; | |
54 | ||
55 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
56 | addr = (*mqd_mem_obj)->gpu_addr; | |
57 | ||
58 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
59 | ||
60 | m->header = 0xC0310800; | |
61 | m->compute_pipelinestat_enable = 1; | |
62 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
63 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
64 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
65 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
66 | ||
67 | /* | |
68 | * Make sure to use the last queue state saved on mqd when the cp | |
69 | * reassigns the queue, so when queue is switched on/off (e.g over | |
70 | * subscription or quantum timeout) the context will be consistent | |
71 | */ | |
72 | m->cp_hqd_persistent_state = | |
73 | DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; | |
74 | ||
75 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
76 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
77 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
78 | ||
79 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; | |
80 | /* Although WinKFD writes this, I suspect it should not be necessary */ | |
81 | m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; | |
82 | ||
83 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
84 | QUANTUM_DURATION(10); | |
85 | ||
86 | /* | |
87 | * Pipe Priority | |
88 | * Identifies the pipe relative priority when this queue is connected | |
89 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
90 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
91 | * 0 = CS_LOW (typically below GFX) | |
92 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
93 | * 2 = CS_HIGH (typically above HP3D) | |
94 | */ | |
95 | m->cp_hqd_pipe_priority = 1; | |
96 | m->cp_hqd_queue_priority = 15; | |
97 | ||
d752f95e JC |
98 | if (q->format == KFD_QUEUE_FORMAT_AQL) |
99 | m->cp_hqd_iq_rptr = AQL_ENABLE; | |
100 | ||
4b8f589b BG |
101 | *mqd = m; |
102 | if (gart_addr != NULL) | |
103 | *gart_addr = addr; | |
104 | retval = mm->update_mqd(mm, m, q); | |
105 | ||
106 | return retval; | |
107 | } | |
108 | ||
109 | static int init_mqd_sdma(struct mqd_manager *mm, void **mqd, | |
110 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
111 | struct queue_properties *q) | |
112 | { | |
113 | int retval; | |
114 | struct cik_sdma_rlc_registers *m; | |
115 | ||
116 | BUG_ON(!mm || !mqd || !mqd_mem_obj); | |
117 | ||
118 | retval = kfd_gtt_sa_allocate(mm->dev, | |
119 | sizeof(struct cik_sdma_rlc_registers), | |
120 | mqd_mem_obj); | |
121 | ||
122 | if (retval != 0) | |
123 | return -ENOMEM; | |
124 | ||
125 | m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr; | |
126 | ||
127 | memset(m, 0, sizeof(struct cik_sdma_rlc_registers)); | |
128 | ||
129 | *mqd = m; | |
130 | if (gart_addr != NULL) | |
131 | *gart_addr = (*mqd_mem_obj)->gpu_addr; | |
132 | ||
133 | retval = mm->update_mqd(mm, m, q); | |
134 | ||
135 | return retval; | |
136 | } | |
137 | ||
138 | static void uninit_mqd(struct mqd_manager *mm, void *mqd, | |
139 | struct kfd_mem_obj *mqd_mem_obj) | |
140 | { | |
141 | BUG_ON(!mm || !mqd); | |
142 | kfd_gtt_sa_free(mm->dev, mqd_mem_obj); | |
143 | } | |
144 | ||
145 | static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
146 | struct kfd_mem_obj *mqd_mem_obj) | |
147 | { | |
148 | BUG_ON(!mm || !mqd); | |
149 | kfd_gtt_sa_free(mm->dev, mqd_mem_obj); | |
150 | } | |
151 | ||
152 | static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, | |
153 | uint32_t queue_id, uint32_t __user *wptr) | |
154 | { | |
cea405b1 XZ |
155 | return mm->dev->kfd2kgd->hqd_load |
156 | (mm->dev->kgd, mqd, pipe_id, queue_id, wptr); | |
4b8f589b BG |
157 | } |
158 | ||
159 | static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
160 | uint32_t pipe_id, uint32_t queue_id, | |
161 | uint32_t __user *wptr) | |
162 | { | |
cea405b1 | 163 | return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd); |
4b8f589b BG |
164 | } |
165 | ||
166 | static int update_mqd(struct mqd_manager *mm, void *mqd, | |
167 | struct queue_properties *q) | |
168 | { | |
169 | struct cik_mqd *m; | |
170 | ||
171 | BUG_ON(!mm || !q || !mqd); | |
172 | ||
173 | pr_debug("kfd: In func %s\n", __func__); | |
174 | ||
175 | m = get_mqd(mqd); | |
176 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
177 | DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; | |
178 | ||
179 | /* | |
180 | * Calculating queue size which is log base 2 of actual queue size -1 | |
181 | * dwords and another -1 for ffs | |
182 | */ | |
183 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
184 | - 1 - 1; | |
185 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
186 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
187 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
188 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
189 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
190 | DOORBELL_OFFSET(q->doorbell_off); | |
191 | ||
192 | m->cp_hqd_vmid = q->vmid; | |
193 | ||
194 | if (q->format == KFD_QUEUE_FORMAT_AQL) { | |
4b8f589b BG |
195 | m->cp_hqd_pq_control |= NO_UPDATE_RPTR; |
196 | } | |
197 | ||
198 | m->cp_hqd_active = 0; | |
199 | q->is_active = false; | |
200 | if (q->queue_size > 0 && | |
201 | q->queue_address != 0 && | |
202 | q->queue_percent > 0) { | |
203 | m->cp_hqd_active = 1; | |
204 | q->is_active = true; | |
205 | } | |
206 | ||
207 | return 0; | |
208 | } | |
209 | ||
210 | static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
211 | struct queue_properties *q) | |
212 | { | |
213 | struct cik_sdma_rlc_registers *m; | |
214 | ||
215 | BUG_ON(!mm || !mqd || !q); | |
216 | ||
217 | m = get_sdma_mqd(mqd); | |
3d30b28b OG |
218 | m->sdma_rlc_rb_cntl = ffs(q->queue_size / sizeof(unsigned int)) << |
219 | SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | | |
220 | q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT | | |
221 | 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | | |
222 | 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; | |
4b8f589b BG |
223 | |
224 | m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); | |
225 | m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); | |
226 | m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
227 | m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
3d30b28b OG |
228 | m->sdma_rlc_doorbell = q->doorbell_off << |
229 | SDMA0_RLC0_DOORBELL__OFFSET__SHIFT | | |
230 | 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT; | |
231 | ||
4b8f589b BG |
232 | m->sdma_rlc_virtual_addr = q->sdma_vm_addr; |
233 | ||
234 | m->sdma_engine_id = q->sdma_engine_id; | |
235 | m->sdma_queue_id = q->sdma_queue_id; | |
236 | ||
237 | q->is_active = false; | |
238 | if (q->queue_size > 0 && | |
239 | q->queue_address != 0 && | |
240 | q->queue_percent > 0) { | |
3d30b28b OG |
241 | m->sdma_rlc_rb_cntl |= |
242 | 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT; | |
243 | ||
4b8f589b BG |
244 | q->is_active = true; |
245 | } | |
246 | ||
247 | return 0; | |
248 | } | |
249 | ||
250 | static int destroy_mqd(struct mqd_manager *mm, void *mqd, | |
251 | enum kfd_preempt_type type, | |
252 | unsigned int timeout, uint32_t pipe_id, | |
253 | uint32_t queue_id) | |
254 | { | |
cea405b1 | 255 | return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, |
4b8f589b BG |
256 | pipe_id, queue_id); |
257 | } | |
258 | ||
259 | /* | |
260 | * preempt type here is ignored because there is only one way | |
261 | * to preempt sdma queue | |
262 | */ | |
263 | static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
264 | enum kfd_preempt_type type, | |
265 | unsigned int timeout, uint32_t pipe_id, | |
266 | uint32_t queue_id) | |
267 | { | |
cea405b1 | 268 | return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout); |
4b8f589b BG |
269 | } |
270 | ||
271 | static bool is_occupied(struct mqd_manager *mm, void *mqd, | |
272 | uint64_t queue_address, uint32_t pipe_id, | |
273 | uint32_t queue_id) | |
274 | { | |
275 | ||
cea405b1 | 276 | return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address, |
4b8f589b BG |
277 | pipe_id, queue_id); |
278 | ||
279 | } | |
280 | ||
281 | static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, | |
282 | uint64_t queue_address, uint32_t pipe_id, | |
283 | uint32_t queue_id) | |
284 | { | |
cea405b1 | 285 | return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd); |
4b8f589b BG |
286 | } |
287 | ||
288 | /* | |
289 | * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. | |
290 | * The HIQ queue in Kaveri is using the same MQD structure as all the user mode | |
291 | * queues but with different initial values. | |
292 | */ | |
293 | ||
294 | static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, | |
295 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
296 | struct queue_properties *q) | |
297 | { | |
298 | uint64_t addr; | |
299 | struct cik_mqd *m; | |
300 | int retval; | |
301 | ||
302 | BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); | |
303 | ||
304 | pr_debug("kfd: In func %s\n", __func__); | |
305 | ||
306 | retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), | |
307 | mqd_mem_obj); | |
308 | ||
309 | if (retval != 0) | |
310 | return -ENOMEM; | |
311 | ||
312 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
313 | addr = (*mqd_mem_obj)->gpu_addr; | |
314 | ||
315 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
316 | ||
317 | m->header = 0xC0310800; | |
318 | m->compute_pipelinestat_enable = 1; | |
319 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
320 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
321 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
322 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
323 | ||
324 | m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | | |
325 | PRELOAD_REQ; | |
326 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
327 | QUANTUM_DURATION(10); | |
328 | ||
329 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
330 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
331 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
332 | ||
333 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; | |
334 | ||
335 | /* | |
336 | * Pipe Priority | |
337 | * Identifies the pipe relative priority when this queue is connected | |
338 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
339 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
340 | * 0 = CS_LOW (typically below GFX) | |
341 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
342 | * 2 = CS_HIGH (typically above HP3D) | |
343 | */ | |
344 | m->cp_hqd_pipe_priority = 1; | |
345 | m->cp_hqd_queue_priority = 15; | |
346 | ||
347 | *mqd = m; | |
348 | if (gart_addr) | |
349 | *gart_addr = addr; | |
350 | retval = mm->update_mqd(mm, m, q); | |
351 | ||
352 | return retval; | |
353 | } | |
354 | ||
355 | static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, | |
356 | struct queue_properties *q) | |
357 | { | |
358 | struct cik_mqd *m; | |
359 | ||
360 | BUG_ON(!mm || !q || !mqd); | |
361 | ||
362 | pr_debug("kfd: In func %s\n", __func__); | |
363 | ||
364 | m = get_mqd(mqd); | |
365 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
366 | DEFAULT_MIN_AVAIL_SIZE | | |
367 | PRIV_STATE | | |
368 | KMD_QUEUE; | |
369 | ||
370 | /* | |
371 | * Calculating queue size which is log base 2 of actual queue | |
372 | * size -1 dwords | |
373 | */ | |
374 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
375 | - 1 - 1; | |
376 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
377 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
378 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
379 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
380 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
381 | DOORBELL_OFFSET(q->doorbell_off); | |
382 | ||
383 | m->cp_hqd_vmid = q->vmid; | |
384 | ||
385 | m->cp_hqd_active = 0; | |
386 | q->is_active = false; | |
387 | if (q->queue_size > 0 && | |
388 | q->queue_address != 0 && | |
389 | q->queue_percent > 0) { | |
390 | m->cp_hqd_active = 1; | |
391 | q->is_active = true; | |
392 | } | |
393 | ||
394 | return 0; | |
395 | } | |
396 | ||
397 | struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) | |
398 | { | |
399 | struct cik_sdma_rlc_registers *m; | |
400 | ||
401 | BUG_ON(!mqd); | |
402 | ||
403 | m = (struct cik_sdma_rlc_registers *)mqd; | |
404 | ||
405 | return m; | |
406 | } | |
407 | ||
408 | struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, | |
409 | struct kfd_dev *dev) | |
410 | { | |
411 | struct mqd_manager *mqd; | |
412 | ||
413 | BUG_ON(!dev); | |
414 | BUG_ON(type >= KFD_MQD_TYPE_MAX); | |
415 | ||
416 | pr_debug("kfd: In func %s\n", __func__); | |
417 | ||
418 | mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); | |
419 | if (!mqd) | |
420 | return NULL; | |
421 | ||
422 | mqd->dev = dev; | |
423 | ||
424 | switch (type) { | |
425 | case KFD_MQD_TYPE_CP: | |
426 | case KFD_MQD_TYPE_COMPUTE: | |
427 | mqd->init_mqd = init_mqd; | |
428 | mqd->uninit_mqd = uninit_mqd; | |
429 | mqd->load_mqd = load_mqd; | |
430 | mqd->update_mqd = update_mqd; | |
431 | mqd->destroy_mqd = destroy_mqd; | |
432 | mqd->is_occupied = is_occupied; | |
433 | break; | |
434 | case KFD_MQD_TYPE_HIQ: | |
435 | mqd->init_mqd = init_mqd_hiq; | |
436 | mqd->uninit_mqd = uninit_mqd; | |
437 | mqd->load_mqd = load_mqd; | |
438 | mqd->update_mqd = update_mqd_hiq; | |
439 | mqd->destroy_mqd = destroy_mqd; | |
440 | mqd->is_occupied = is_occupied; | |
441 | break; | |
442 | case KFD_MQD_TYPE_SDMA: | |
443 | mqd->init_mqd = init_mqd_sdma; | |
444 | mqd->uninit_mqd = uninit_mqd_sdma; | |
445 | mqd->load_mqd = load_mqd_sdma; | |
446 | mqd->update_mqd = update_mqd_sdma; | |
447 | mqd->destroy_mqd = destroy_mqd_sdma; | |
448 | mqd->is_occupied = is_occupied_sdma; | |
449 | break; | |
450 | default: | |
451 | kfree(mqd); | |
452 | return NULL; | |
453 | } | |
454 | ||
455 | return mqd; | |
456 | } | |
457 |