Commit | Line | Data |
---|---|---|
6e99df57 BG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
24 | #include <linux/printk.h> | |
25 | #include <linux/slab.h> | |
26 | #include "kfd_priv.h" | |
27 | #include "kfd_mqd_manager.h" | |
28 | #include "cik_regs.h" | |
29 | #include "../../radeon/cik_reg.h" | |
30 | ||
31 | inline void busy_wait(unsigned long ms) | |
32 | { | |
33 | while (time_before(jiffies, ms)) | |
34 | cpu_relax(); | |
35 | } | |
36 | ||
37 | static inline struct cik_mqd *get_mqd(void *mqd) | |
38 | { | |
39 | return (struct cik_mqd *)mqd; | |
40 | } | |
41 | ||
42 | static int init_mqd(struct mqd_manager *mm, void **mqd, | |
43 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
44 | struct queue_properties *q) | |
45 | { | |
46 | uint64_t addr; | |
47 | struct cik_mqd *m; | |
48 | int retval; | |
49 | ||
50 | BUG_ON(!mm || !q || !mqd); | |
51 | ||
52 | pr_debug("kfd: In func %s\n", __func__); | |
53 | ||
54 | retval = kfd2kgd->allocate_mem(mm->dev->kgd, | |
55 | sizeof(struct cik_mqd), | |
56 | 256, | |
57 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
58 | (struct kgd_mem **) mqd_mem_obj); | |
59 | ||
60 | if (retval != 0) | |
61 | return -ENOMEM; | |
62 | ||
63 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
64 | addr = (*mqd_mem_obj)->gpu_addr; | |
65 | ||
66 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
67 | ||
68 | m->header = 0xC0310800; | |
69 | m->compute_pipelinestat_enable = 1; | |
70 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
71 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
72 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
73 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
74 | ||
75 | /* | |
76 | * Make sure to use the last queue state saved on mqd when the cp | |
77 | * reassigns the queue, so when queue is switched on/off (e.g over | |
78 | * subscription or quantum timeout) the context will be consistent | |
79 | */ | |
80 | m->cp_hqd_persistent_state = | |
81 | DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; | |
82 | ||
83 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
84 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
85 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
86 | ||
87 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; | |
88 | /* Although WinKFD writes this, I suspect it should not be necessary */ | |
89 | m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; | |
90 | ||
91 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
92 | QUANTUM_DURATION(10); | |
93 | ||
94 | /* | |
95 | * Pipe Priority | |
96 | * Identifies the pipe relative priority when this queue is connected | |
97 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
98 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
99 | * 0 = CS_LOW (typically below GFX) | |
100 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
101 | * 2 = CS_HIGH (typically above HP3D) | |
102 | */ | |
103 | m->cp_hqd_pipe_priority = 1; | |
104 | m->cp_hqd_queue_priority = 15; | |
105 | ||
106 | *mqd = m; | |
107 | if (gart_addr != NULL) | |
108 | *gart_addr = addr; | |
109 | retval = mm->update_mqd(mm, m, q); | |
110 | ||
111 | return retval; | |
112 | } | |
113 | ||
114 | static void uninit_mqd(struct mqd_manager *mm, void *mqd, | |
115 | struct kfd_mem_obj *mqd_mem_obj) | |
116 | { | |
117 | BUG_ON(!mm || !mqd); | |
118 | kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj); | |
119 | } | |
120 | ||
121 | static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, | |
122 | uint32_t queue_id, uint32_t __user *wptr) | |
123 | { | |
124 | return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr); | |
125 | ||
126 | } | |
127 | ||
128 | static int update_mqd(struct mqd_manager *mm, void *mqd, | |
129 | struct queue_properties *q) | |
130 | { | |
131 | struct cik_mqd *m; | |
132 | ||
133 | BUG_ON(!mm || !q || !mqd); | |
134 | ||
135 | pr_debug("kfd: In func %s\n", __func__); | |
136 | ||
137 | m = get_mqd(mqd); | |
138 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
139 | DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; | |
140 | ||
141 | /* | |
142 | * Calculating queue size which is log base 2 of actual queue size -1 | |
143 | * dwords and another -1 for ffs | |
144 | */ | |
145 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
146 | - 1 - 1; | |
147 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
148 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
149 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
150 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
151 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
152 | DOORBELL_OFFSET(q->doorbell_off); | |
153 | ||
154 | m->cp_hqd_vmid = q->vmid; | |
155 | ||
156 | if (q->format == KFD_QUEUE_FORMAT_AQL) { | |
157 | m->cp_hqd_iq_rptr = AQL_ENABLE; | |
158 | m->cp_hqd_pq_control |= NO_UPDATE_RPTR; | |
159 | } | |
160 | ||
161 | m->cp_hqd_active = 0; | |
162 | q->is_active = false; | |
163 | if (q->queue_size > 0 && | |
164 | q->queue_address != 0 && | |
165 | q->queue_percent > 0) { | |
166 | m->cp_hqd_active = 1; | |
167 | q->is_active = true; | |
168 | } | |
169 | ||
170 | return 0; | |
171 | } | |
172 | ||
173 | static int destroy_mqd(struct mqd_manager *mm, void *mqd, | |
174 | enum kfd_preempt_type type, | |
175 | unsigned int timeout, uint32_t pipe_id, | |
176 | uint32_t queue_id) | |
177 | { | |
178 | return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, | |
179 | pipe_id, queue_id); | |
180 | } | |
181 | ||
182 | bool is_occupied(struct mqd_manager *mm, void *mqd, | |
183 | uint64_t queue_address, uint32_t pipe_id, | |
184 | uint32_t queue_id) | |
185 | { | |
186 | ||
187 | return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address, | |
188 | pipe_id, queue_id); | |
189 | ||
190 | } | |
191 | ||
192 | /* | |
193 | * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. | |
194 | * The HIQ queue in Kaveri is using the same MQD structure as all the user mode | |
195 | * queues but with different initial values. | |
196 | */ | |
197 | ||
198 | static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, | |
199 | struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, | |
200 | struct queue_properties *q) | |
201 | { | |
202 | uint64_t addr; | |
203 | struct cik_mqd *m; | |
204 | int retval; | |
205 | ||
206 | BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); | |
207 | ||
208 | pr_debug("kfd: In func %s\n", __func__); | |
209 | ||
210 | retval = kfd2kgd->allocate_mem(mm->dev->kgd, | |
211 | sizeof(struct cik_mqd), | |
212 | 256, | |
213 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE, | |
214 | (struct kgd_mem **) mqd_mem_obj); | |
215 | ||
216 | if (retval != 0) | |
217 | return -ENOMEM; | |
218 | ||
219 | m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; | |
220 | addr = (*mqd_mem_obj)->gpu_addr; | |
221 | ||
222 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
223 | ||
224 | m->header = 0xC0310800; | |
225 | m->compute_pipelinestat_enable = 1; | |
226 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
227 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
228 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
229 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
230 | ||
231 | m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | | |
232 | PRELOAD_REQ; | |
233 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | | |
234 | QUANTUM_DURATION(10); | |
235 | ||
236 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
237 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
238 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
239 | ||
240 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; | |
241 | ||
242 | /* | |
243 | * Pipe Priority | |
244 | * Identifies the pipe relative priority when this queue is connected | |
245 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
246 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
247 | * 0 = CS_LOW (typically below GFX) | |
248 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
249 | * 2 = CS_HIGH (typically above HP3D) | |
250 | */ | |
251 | m->cp_hqd_pipe_priority = 1; | |
252 | m->cp_hqd_queue_priority = 15; | |
253 | ||
254 | *mqd = m; | |
255 | if (gart_addr) | |
256 | *gart_addr = addr; | |
257 | retval = mm->update_mqd(mm, m, q); | |
258 | ||
259 | return retval; | |
260 | } | |
261 | ||
262 | static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, | |
263 | struct queue_properties *q) | |
264 | { | |
265 | struct cik_mqd *m; | |
266 | ||
267 | BUG_ON(!mm || !q || !mqd); | |
268 | ||
269 | pr_debug("kfd: In func %s\n", __func__); | |
270 | ||
271 | m = get_mqd(mqd); | |
272 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
273 | DEFAULT_MIN_AVAIL_SIZE | | |
274 | PRIV_STATE | | |
275 | KMD_QUEUE; | |
276 | ||
277 | /* | |
278 | * Calculating queue size which is log base 2 of actual queue | |
279 | * size -1 dwords | |
280 | */ | |
281 | m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) | |
282 | - 1 - 1; | |
283 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
284 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
285 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
286 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
287 | m->cp_hqd_pq_doorbell_control = DOORBELL_EN | | |
288 | DOORBELL_OFFSET(q->doorbell_off); | |
289 | ||
290 | m->cp_hqd_vmid = q->vmid; | |
291 | ||
292 | m->cp_hqd_active = 0; | |
293 | q->is_active = false; | |
294 | if (q->queue_size > 0 && | |
295 | q->queue_address != 0 && | |
296 | q->queue_percent > 0) { | |
297 | m->cp_hqd_active = 1; | |
298 | q->is_active = true; | |
299 | } | |
300 | ||
301 | return 0; | |
302 | } | |
303 | ||
304 | struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, | |
305 | struct kfd_dev *dev) | |
306 | { | |
307 | struct mqd_manager *mqd; | |
308 | ||
309 | BUG_ON(!dev); | |
310 | BUG_ON(type >= KFD_MQD_TYPE_MAX); | |
311 | ||
312 | pr_debug("kfd: In func %s\n", __func__); | |
313 | ||
314 | mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); | |
315 | if (!mqd) | |
316 | return NULL; | |
317 | ||
318 | mqd->dev = dev; | |
319 | ||
320 | switch (type) { | |
321 | case KFD_MQD_TYPE_CIK_CP: | |
322 | case KFD_MQD_TYPE_CIK_COMPUTE: | |
323 | mqd->init_mqd = init_mqd; | |
324 | mqd->uninit_mqd = uninit_mqd; | |
325 | mqd->load_mqd = load_mqd; | |
326 | mqd->update_mqd = update_mqd; | |
327 | mqd->destroy_mqd = destroy_mqd; | |
328 | mqd->is_occupied = is_occupied; | |
329 | break; | |
330 | case KFD_MQD_TYPE_CIK_HIQ: | |
331 | mqd->init_mqd = init_mqd_hiq; | |
332 | mqd->uninit_mqd = uninit_mqd; | |
333 | mqd->load_mqd = load_mqd; | |
334 | mqd->update_mqd = update_mqd_hiq; | |
335 | mqd->destroy_mqd = destroy_mqd; | |
336 | mqd->is_occupied = is_occupied; | |
337 | break; | |
338 | default: | |
339 | kfree(mqd); | |
340 | return NULL; | |
341 | } | |
342 | ||
343 | return mqd; | |
344 | } | |
345 | ||
346 | /* SDMA queues should be implemented here when the cp will supports them */ |