Commit | Line | Data |
---|---|---|
6e99df57 BG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
24 | #include <linux/printk.h> | |
25 | #include <linux/slab.h> | |
26 | #include "kfd_priv.h" | |
27 | #include "kfd_mqd_manager.h" | |
28 | #include "cik_regs.h" | |
77669eb8 | 29 | #include "../../radeon/cikd.h" |
6e99df57 BG |
30 | #include "../../radeon/cik_reg.h" |
31 | ||
32 | inline void busy_wait(unsigned long ms) | |
33 | { | |
34 | while (time_before(jiffies, ms)) | |
35 | cpu_relax(); | |
36 | } | |
37 | ||
38 | static inline struct cik_mqd *get_mqd(void *mqd) | |
39 | { | |
40 | return (struct cik_mqd *)mqd; | |
41 | } | |
42 | ||
43 | static int init_mqd(struct mqd_manager *mm, void **mqd, | |
44 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
45 | struct queue_properties *q) | |
46 | { | |
47 | uint64_t addr; | |
48 | struct cik_mqd *m; | |
49 | int retval; | |
50 | ||
51 | BUG_ON(!mm || !q || !mqd); | |
52 | ||
53 | pr_debug("kfd: In func %s\n", __func__); | |
54 | ||
a86aa3ca OG |
55 | retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), |
56 | mqd_mem_obj); | |
6e99df57 BG |
57 | |
58 | if (retval != 0) | |
59 | return -ENOMEM; | |
60 | ||
61 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
62 | addr = (*mqd_mem_obj)->gpu_addr; | |
63 | ||
64 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
65 | ||
66 | m->header = 0xC0310800; | |
67 | m->compute_pipelinestat_enable = 1; | |
68 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
69 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
70 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
71 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
72 | ||
73 | /* | |
74 | * Make sure to use the last queue state saved on mqd when the cp | |
75 | * reassigns the queue, so when queue is switched on/off (e.g over | |
76 | * subscription or quantum timeout) the context will be consistent | |
77 | */ | |
78 | m->cp_hqd_persistent_state = | |
79 | DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; | |
80 | ||
81 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
82 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
83 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
84 | ||
85 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; | |
86 | /* Although WinKFD writes this, I suspect it should not be necessary */ | |
87 | m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; | |
88 | ||
89 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
90 | QUANTUM_DURATION(10); | |
91 | ||
92 | /* | |
93 | * Pipe Priority | |
94 | * Identifies the pipe relative priority when this queue is connected | |
95 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
96 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
97 | * 0 = CS_LOW (typically below GFX) | |
98 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
99 | * 2 = CS_HIGH (typically above HP3D) | |
100 | */ | |
101 | m->cp_hqd_pipe_priority = 1; | |
102 | m->cp_hqd_queue_priority = 15; | |
103 | ||
104 | *mqd = m; | |
105 | if (gart_addr != NULL) | |
106 | *gart_addr = addr; | |
107 | retval = mm->update_mqd(mm, m, q); | |
108 | ||
109 | return retval; | |
110 | } | |
111 | ||
77669eb8 BG |
112 | static int init_mqd_sdma(struct mqd_manager *mm, void **mqd, |
113 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
114 | struct queue_properties *q) | |
115 | { | |
116 | int retval; | |
117 | struct cik_sdma_rlc_registers *m; | |
118 | ||
119 | BUG_ON(!mm || !mqd || !mqd_mem_obj); | |
120 | ||
a86aa3ca | 121 | retval = kfd_gtt_sa_allocate(mm->dev, |
77669eb8 | 122 | sizeof(struct cik_sdma_rlc_registers), |
a86aa3ca | 123 | mqd_mem_obj); |
77669eb8 BG |
124 | |
125 | if (retval != 0) | |
126 | return -ENOMEM; | |
127 | ||
128 | m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr; | |
129 | ||
130 | memset(m, 0, sizeof(struct cik_sdma_rlc_registers)); | |
131 | ||
132 | *mqd = m; | |
133 | if (gart_addr != NULL) | |
134 | *gart_addr = (*mqd_mem_obj)->gpu_addr; | |
135 | ||
136 | retval = mm->update_mqd(mm, m, q); | |
137 | ||
138 | return retval; | |
139 | } | |
140 | ||
6e99df57 BG |
141 | static void uninit_mqd(struct mqd_manager *mm, void *mqd, |
142 | struct kfd_mem_obj *mqd_mem_obj) | |
143 | { | |
144 | BUG_ON(!mm || !mqd); | |
a86aa3ca | 145 | kfd_gtt_sa_free(mm->dev, mqd_mem_obj); |
6e99df57 BG |
146 | } |
147 | ||
77669eb8 BG |
148 | static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd, |
149 | struct kfd_mem_obj *mqd_mem_obj) | |
150 | { | |
151 | BUG_ON(!mm || !mqd); | |
a86aa3ca | 152 | kfd_gtt_sa_free(mm->dev, mqd_mem_obj); |
77669eb8 BG |
153 | } |
154 | ||
6e99df57 BG |
155 | static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, |
156 | uint32_t queue_id, uint32_t __user *wptr) | |
157 | { | |
158 | return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr); | |
77669eb8 | 159 | } |
6e99df57 | 160 | |
77669eb8 BG |
161 | static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, |
162 | uint32_t pipe_id, uint32_t queue_id, | |
163 | uint32_t __user *wptr) | |
164 | { | |
165 | return kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd); | |
6e99df57 BG |
166 | } |
167 | ||
168 | static int update_mqd(struct mqd_manager *mm, void *mqd, | |
169 | struct queue_properties *q) | |
170 | { | |
171 | struct cik_mqd *m; | |
172 | ||
173 | BUG_ON(!mm || !q || !mqd); | |
174 | ||
175 | pr_debug("kfd: In func %s\n", __func__); | |
176 | ||
177 | m = get_mqd(mqd); | |
178 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
179 | DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; | |
180 | ||
181 | /* | |
182 | * Calculating queue size which is log base 2 of actual queue size -1 | |
183 | * dwords and another -1 for ffs | |
184 | */ | |
185 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
186 | - 1 - 1; | |
187 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
188 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
189 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
190 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
191 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
192 | DOORBELL_OFFSET(q->doorbell_off); | |
193 | ||
194 | m->cp_hqd_vmid = q->vmid; | |
195 | ||
196 | if (q->format == KFD_QUEUE_FORMAT_AQL) { | |
197 | m->cp_hqd_iq_rptr = AQL_ENABLE; | |
198 | m->cp_hqd_pq_control |= NO_UPDATE_RPTR; | |
199 | } | |
200 | ||
201 | m->cp_hqd_active = 0; | |
202 | q->is_active = false; | |
203 | if (q->queue_size > 0 && | |
204 | q->queue_address != 0 && | |
205 | q->queue_percent > 0) { | |
206 | m->cp_hqd_active = 1; | |
207 | q->is_active = true; | |
208 | } | |
209 | ||
210 | return 0; | |
211 | } | |
212 | ||
77669eb8 BG |
213 | static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, |
214 | struct queue_properties *q) | |
215 | { | |
216 | struct cik_sdma_rlc_registers *m; | |
217 | ||
218 | BUG_ON(!mm || !mqd || !q); | |
219 | ||
220 | m = get_sdma_mqd(mqd); | |
221 | m->sdma_rlc_rb_cntl = | |
222 | SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) | | |
223 | SDMA_RB_VMID(q->vmid) | | |
224 | SDMA_RPTR_WRITEBACK_ENABLE | | |
225 | SDMA_RPTR_WRITEBACK_TIMER(6); | |
226 | ||
227 | m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); | |
228 | m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); | |
229 | m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
230 | m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
231 | m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE; | |
232 | m->sdma_rlc_virtual_addr = q->sdma_vm_addr; | |
233 | ||
234 | m->sdma_engine_id = q->sdma_engine_id; | |
235 | m->sdma_queue_id = q->sdma_queue_id; | |
236 | ||
237 | q->is_active = false; | |
238 | if (q->queue_size > 0 && | |
239 | q->queue_address != 0 && | |
240 | q->queue_percent > 0) { | |
241 | m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE; | |
242 | q->is_active = true; | |
243 | } | |
244 | ||
245 | return 0; | |
246 | } | |
247 | ||
6e99df57 BG |
248 | static int destroy_mqd(struct mqd_manager *mm, void *mqd, |
249 | enum kfd_preempt_type type, | |
250 | unsigned int timeout, uint32_t pipe_id, | |
251 | uint32_t queue_id) | |
252 | { | |
253 | return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, | |
254 | pipe_id, queue_id); | |
255 | } | |
256 | ||
77669eb8 BG |
257 | /* |
258 | * preempt type here is ignored because there is only one way | |
259 | * to preempt sdma queue | |
260 | */ | |
261 | static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
262 | enum kfd_preempt_type type, | |
263 | unsigned int timeout, uint32_t pipe_id, | |
264 | uint32_t queue_id) | |
265 | { | |
266 | return kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout); | |
267 | } | |
268 | ||
20981e68 OG |
269 | static bool is_occupied(struct mqd_manager *mm, void *mqd, |
270 | uint64_t queue_address, uint32_t pipe_id, | |
271 | uint32_t queue_id) | |
6e99df57 BG |
272 | { |
273 | ||
274 | return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address, | |
275 | pipe_id, queue_id); | |
276 | ||
277 | } | |
278 | ||
77669eb8 BG |
279 | static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, |
280 | uint64_t queue_address, uint32_t pipe_id, | |
281 | uint32_t queue_id) | |
282 | { | |
283 | return kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd); | |
284 | } | |
285 | ||
6e99df57 BG |
286 | /* |
287 | * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. | |
288 | * The HIQ queue in Kaveri is using the same MQD structure as all the user mode | |
289 | * queues but with different initial values. | |
290 | */ | |
291 | ||
292 | static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, | |
293 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
294 | struct queue_properties *q) | |
295 | { | |
296 | uint64_t addr; | |
297 | struct cik_mqd *m; | |
298 | int retval; | |
299 | ||
300 | BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); | |
301 | ||
302 | pr_debug("kfd: In func %s\n", __func__); | |
303 | ||
a86aa3ca OG |
304 | retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), |
305 | mqd_mem_obj); | |
6e99df57 BG |
306 | |
307 | if (retval != 0) | |
308 | return -ENOMEM; | |
309 | ||
310 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
311 | addr = (*mqd_mem_obj)->gpu_addr; | |
312 | ||
313 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
314 | ||
315 | m->header = 0xC0310800; | |
316 | m->compute_pipelinestat_enable = 1; | |
317 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
318 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
319 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
320 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
321 | ||
322 | m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | | |
323 | PRELOAD_REQ; | |
324 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
325 | QUANTUM_DURATION(10); | |
326 | ||
327 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
328 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
329 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
330 | ||
331 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; | |
332 | ||
333 | /* | |
334 | * Pipe Priority | |
335 | * Identifies the pipe relative priority when this queue is connected | |
336 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
337 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
338 | * 0 = CS_LOW (typically below GFX) | |
339 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
340 | * 2 = CS_HIGH (typically above HP3D) | |
341 | */ | |
342 | m->cp_hqd_pipe_priority = 1; | |
343 | m->cp_hqd_queue_priority = 15; | |
344 | ||
345 | *mqd = m; | |
346 | if (gart_addr) | |
347 | *gart_addr = addr; | |
348 | retval = mm->update_mqd(mm, m, q); | |
349 | ||
350 | return retval; | |
351 | } | |
352 | ||
353 | static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, | |
354 | struct queue_properties *q) | |
355 | { | |
356 | struct cik_mqd *m; | |
357 | ||
358 | BUG_ON(!mm || !q || !mqd); | |
359 | ||
360 | pr_debug("kfd: In func %s\n", __func__); | |
361 | ||
362 | m = get_mqd(mqd); | |
363 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
364 | DEFAULT_MIN_AVAIL_SIZE | | |
365 | PRIV_STATE | | |
366 | KMD_QUEUE; | |
367 | ||
368 | /* | |
369 | * Calculating queue size which is log base 2 of actual queue | |
370 | * size -1 dwords | |
371 | */ | |
372 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
373 | - 1 - 1; | |
374 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
375 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
376 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
377 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
378 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
379 | DOORBELL_OFFSET(q->doorbell_off); | |
380 | ||
381 | m->cp_hqd_vmid = q->vmid; | |
382 | ||
383 | m->cp_hqd_active = 0; | |
384 | q->is_active = false; | |
385 | if (q->queue_size > 0 && | |
386 | q->queue_address != 0 && | |
387 | q->queue_percent > 0) { | |
388 | m->cp_hqd_active = 1; | |
389 | q->is_active = true; | |
390 | } | |
391 | ||
392 | return 0; | |
393 | } | |
394 | ||
77669eb8 BG |
395 | /* |
396 | * SDMA MQD Implementation | |
397 | */ | |
398 | ||
399 | struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) | |
400 | { | |
401 | struct cik_sdma_rlc_registers *m; | |
402 | ||
403 | BUG_ON(!mqd); | |
404 | ||
405 | m = (struct cik_sdma_rlc_registers *)mqd; | |
406 | ||
407 | return m; | |
408 | } | |
409 | ||
6e99df57 BG |
410 | struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, |
411 | struct kfd_dev *dev) | |
412 | { | |
413 | struct mqd_manager *mqd; | |
414 | ||
415 | BUG_ON(!dev); | |
416 | BUG_ON(type >= KFD_MQD_TYPE_MAX); | |
417 | ||
418 | pr_debug("kfd: In func %s\n", __func__); | |
419 | ||
420 | mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); | |
421 | if (!mqd) | |
422 | return NULL; | |
423 | ||
424 | mqd->dev = dev; | |
425 | ||
426 | switch (type) { | |
427 | case KFD_MQD_TYPE_CIK_CP: | |
428 | case KFD_MQD_TYPE_CIK_COMPUTE: | |
429 | mqd->init_mqd = init_mqd; | |
430 | mqd->uninit_mqd = uninit_mqd; | |
431 | mqd->load_mqd = load_mqd; | |
432 | mqd->update_mqd = update_mqd; | |
433 | mqd->destroy_mqd = destroy_mqd; | |
434 | mqd->is_occupied = is_occupied; | |
435 | break; | |
436 | case KFD_MQD_TYPE_CIK_HIQ: | |
437 | mqd->init_mqd = init_mqd_hiq; | |
438 | mqd->uninit_mqd = uninit_mqd; | |
439 | mqd->load_mqd = load_mqd; | |
440 | mqd->update_mqd = update_mqd_hiq; | |
441 | mqd->destroy_mqd = destroy_mqd; | |
442 | mqd->is_occupied = is_occupied; | |
443 | break; | |
77669eb8 BG |
444 | case KFD_MQD_TYPE_CIK_SDMA: |
445 | mqd->init_mqd = init_mqd_sdma; | |
446 | mqd->uninit_mqd = uninit_mqd_sdma; | |
447 | mqd->load_mqd = load_mqd_sdma; | |
448 | mqd->update_mqd = update_mqd_sdma; | |
449 | mqd->destroy_mqd = destroy_mqd_sdma; | |
450 | mqd->is_occupied = is_occupied_sdma; | |
451 | break; | |
6e99df57 BG |
452 | default: |
453 | kfree(mqd); | |
454 | return NULL; | |
455 | } | |
456 | ||
457 | return mqd; | |
458 | } | |
459 | ||
460 | /* SDMA queues should be implemented here when the cp will supports them */ |