Commit | Line | Data |
---|---|---|
4a488a7a OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #include <linux/device.h> | |
24 | #include <linux/export.h> | |
25 | #include <linux/err.h> | |
26 | #include <linux/fs.h> | |
27 | #include <linux/sched.h> | |
28 | #include <linux/slab.h> | |
29 | #include <linux/uaccess.h> | |
30 | #include <linux/compat.h> | |
31 | #include <uapi/linux/kfd_ioctl.h> | |
32 | #include <linux/time.h> | |
33 | #include <linux/mm.h> | |
4a488a7a OG |
34 | #include <uapi/asm-generic/mman-common.h> |
35 | #include <asm/processor.h> | |
36 | #include "kfd_priv.h" | |
41a286fa | 37 | #include "kfd_device_queue_manager.h" |
037ed9a2 | 38 | #include "kfd_dbgmgr.h" |
4a488a7a OG |
39 | |
40 | static long kfd_ioctl(struct file *, unsigned int, unsigned long); | |
41 | static int kfd_open(struct inode *, struct file *); | |
19f6d2a6 | 42 | static int kfd_mmap(struct file *, struct vm_area_struct *); |
4a488a7a OG |
43 | |
44 | static const char kfd_dev_name[] = "kfd"; | |
45 | ||
46 | static const struct file_operations kfd_fops = { | |
47 | .owner = THIS_MODULE, | |
48 | .unlocked_ioctl = kfd_ioctl, | |
49 | .compat_ioctl = kfd_ioctl, | |
50 | .open = kfd_open, | |
19f6d2a6 | 51 | .mmap = kfd_mmap, |
4a488a7a OG |
52 | }; |
53 | ||
54 | static int kfd_char_dev_major = -1; | |
55 | static struct class *kfd_class; | |
56 | struct device *kfd_device; | |
57 | ||
58 | int kfd_chardev_init(void) | |
59 | { | |
60 | int err = 0; | |
61 | ||
62 | kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); | |
63 | err = kfd_char_dev_major; | |
64 | if (err < 0) | |
65 | goto err_register_chrdev; | |
66 | ||
67 | kfd_class = class_create(THIS_MODULE, kfd_dev_name); | |
68 | err = PTR_ERR(kfd_class); | |
69 | if (IS_ERR(kfd_class)) | |
70 | goto err_class_create; | |
71 | ||
72 | kfd_device = device_create(kfd_class, NULL, | |
73 | MKDEV(kfd_char_dev_major, 0), | |
74 | NULL, kfd_dev_name); | |
75 | err = PTR_ERR(kfd_device); | |
76 | if (IS_ERR(kfd_device)) | |
77 | goto err_device_create; | |
78 | ||
79 | return 0; | |
80 | ||
81 | err_device_create: | |
82 | class_destroy(kfd_class); | |
83 | err_class_create: | |
84 | unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | |
85 | err_register_chrdev: | |
86 | return err; | |
87 | } | |
88 | ||
89 | void kfd_chardev_exit(void) | |
90 | { | |
91 | device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); | |
92 | class_destroy(kfd_class); | |
93 | unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | |
94 | } | |
95 | ||
96 | struct device *kfd_chardev(void) | |
97 | { | |
98 | return kfd_device; | |
99 | } | |
100 | ||
101 | ||
102 | static int kfd_open(struct inode *inode, struct file *filep) | |
103 | { | |
19f6d2a6 | 104 | struct kfd_process *process; |
a18069c1 | 105 | bool is_32bit_user_mode; |
19f6d2a6 | 106 | |
4a488a7a OG |
107 | if (iminor(inode) != 0) |
108 | return -ENODEV; | |
109 | ||
a18069c1 OG |
110 | is_32bit_user_mode = is_compat_task(); |
111 | ||
112 | if (is_32bit_user_mode == true) { | |
113 | dev_warn(kfd_device, | |
114 | "Process %d (32-bit) failed to open /dev/kfd\n" | |
115 | "32-bit processes are not supported by amdkfd\n", | |
116 | current->pid); | |
117 | return -EPERM; | |
118 | } | |
119 | ||
19f6d2a6 OG |
120 | process = kfd_create_process(current); |
121 | if (IS_ERR(process)) | |
122 | return PTR_ERR(process); | |
123 | ||
19f6d2a6 OG |
124 | dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", |
125 | process->pasid, process->is_32bit_user_mode); | |
126 | ||
4a488a7a OG |
127 | return 0; |
128 | } | |
129 | ||
524a6404 OG |
130 | static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, |
131 | void *data) | |
4a488a7a | 132 | { |
524a6404 | 133 | struct kfd_ioctl_get_version_args *args = data; |
ecd5c982 OG |
134 | int err = 0; |
135 | ||
524a6404 OG |
136 | args->major_version = KFD_IOCTL_MAJOR_VERSION; |
137 | args->minor_version = KFD_IOCTL_MINOR_VERSION; | |
ecd5c982 OG |
138 | |
139 | return err; | |
4a488a7a OG |
140 | } |
141 | ||
39b027d9 OG |
142 | static int set_queue_properties_from_user(struct queue_properties *q_properties, |
143 | struct kfd_ioctl_create_queue_args *args) | |
144 | { | |
145 | if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | |
146 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | |
147 | return -EINVAL; | |
148 | } | |
149 | ||
150 | if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { | |
151 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | |
152 | return -EINVAL; | |
153 | } | |
154 | ||
155 | if ((args->ring_base_address) && | |
4307d8f6 OG |
156 | (!access_ok(VERIFY_WRITE, |
157 | (const void __user *) args->ring_base_address, | |
158 | sizeof(uint64_t)))) { | |
39b027d9 OG |
159 | pr_err("kfd: can't access ring base address\n"); |
160 | return -EFAULT; | |
161 | } | |
162 | ||
163 | if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { | |
164 | pr_err("kfd: ring size must be a power of 2 or 0\n"); | |
165 | return -EINVAL; | |
166 | } | |
167 | ||
4307d8f6 OG |
168 | if (!access_ok(VERIFY_WRITE, |
169 | (const void __user *) args->read_pointer_address, | |
170 | sizeof(uint32_t))) { | |
39b027d9 OG |
171 | pr_err("kfd: can't access read pointer\n"); |
172 | return -EFAULT; | |
173 | } | |
174 | ||
4307d8f6 OG |
175 | if (!access_ok(VERIFY_WRITE, |
176 | (const void __user *) args->write_pointer_address, | |
177 | sizeof(uint32_t))) { | |
39b027d9 OG |
178 | pr_err("kfd: can't access write pointer\n"); |
179 | return -EFAULT; | |
180 | } | |
181 | ||
0b3674ae OG |
182 | if (args->eop_buffer_address && |
183 | !access_ok(VERIFY_WRITE, | |
184 | (const void __user *) args->eop_buffer_address, | |
185 | sizeof(uint32_t))) { | |
ff3d04a1 BG |
186 | pr_debug("kfd: can't access eop buffer"); |
187 | return -EFAULT; | |
188 | } | |
189 | ||
0b3674ae OG |
190 | if (args->ctx_save_restore_address && |
191 | !access_ok(VERIFY_WRITE, | |
192 | (const void __user *) args->ctx_save_restore_address, | |
193 | sizeof(uint32_t))) { | |
ff3d04a1 BG |
194 | pr_debug("kfd: can't access ctx save restore buffer"); |
195 | return -EFAULT; | |
196 | } | |
197 | ||
39b027d9 OG |
198 | q_properties->is_interop = false; |
199 | q_properties->queue_percent = args->queue_percentage; | |
200 | q_properties->priority = args->queue_priority; | |
201 | q_properties->queue_address = args->ring_base_address; | |
202 | q_properties->queue_size = args->ring_size; | |
203 | q_properties->read_ptr = (uint32_t *) args->read_pointer_address; | |
204 | q_properties->write_ptr = (uint32_t *) args->write_pointer_address; | |
ff3d04a1 BG |
205 | q_properties->eop_ring_buffer_address = args->eop_buffer_address; |
206 | q_properties->eop_ring_buffer_size = args->eop_buffer_size; | |
207 | q_properties->ctx_save_restore_area_address = | |
208 | args->ctx_save_restore_address; | |
209 | q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size; | |
39b027d9 OG |
210 | if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || |
211 | args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | |
212 | q_properties->type = KFD_QUEUE_TYPE_COMPUTE; | |
3385f9dd BG |
213 | else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) |
214 | q_properties->type = KFD_QUEUE_TYPE_SDMA; | |
39b027d9 OG |
215 | else |
216 | return -ENOTSUPP; | |
217 | ||
218 | if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | |
219 | q_properties->format = KFD_QUEUE_FORMAT_AQL; | |
220 | else | |
221 | q_properties->format = KFD_QUEUE_FORMAT_PM4; | |
222 | ||
223 | pr_debug("Queue Percentage (%d, %d)\n", | |
224 | q_properties->queue_percent, args->queue_percentage); | |
225 | ||
226 | pr_debug("Queue Priority (%d, %d)\n", | |
227 | q_properties->priority, args->queue_priority); | |
228 | ||
229 | pr_debug("Queue Address (0x%llX, 0x%llX)\n", | |
230 | q_properties->queue_address, args->ring_base_address); | |
231 | ||
232 | pr_debug("Queue Size (0x%llX, %u)\n", | |
233 | q_properties->queue_size, args->ring_size); | |
234 | ||
235 | pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n", | |
236 | (uint64_t) q_properties->read_ptr, | |
237 | (uint64_t) q_properties->write_ptr); | |
238 | ||
239 | pr_debug("Queue Format (%d)\n", q_properties->format); | |
240 | ||
ff3d04a1 BG |
241 | pr_debug("Queue EOP (0x%llX)\n", q_properties->eop_ring_buffer_address); |
242 | ||
243 | pr_debug("Queue CTX save arex (0x%llX)\n", | |
244 | q_properties->ctx_save_restore_area_address); | |
245 | ||
39b027d9 OG |
246 | return 0; |
247 | } | |
248 | ||
524a6404 OG |
249 | static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, |
250 | void *data) | |
4a488a7a | 251 | { |
524a6404 | 252 | struct kfd_ioctl_create_queue_args *args = data; |
39b027d9 OG |
253 | struct kfd_dev *dev; |
254 | int err = 0; | |
255 | unsigned int queue_id; | |
256 | struct kfd_process_device *pdd; | |
257 | struct queue_properties q_properties; | |
258 | ||
259 | memset(&q_properties, 0, sizeof(struct queue_properties)); | |
260 | ||
39b027d9 OG |
261 | pr_debug("kfd: creating queue ioctl\n"); |
262 | ||
524a6404 | 263 | err = set_queue_properties_from_user(&q_properties, args); |
39b027d9 OG |
264 | if (err) |
265 | return err; | |
266 | ||
281d1bbd | 267 | pr_debug("kfd: looking for gpu id 0x%x\n", args->gpu_id); |
524a6404 | 268 | dev = kfd_device_by_id(args->gpu_id); |
ff3d04a1 | 269 | if (dev == NULL) { |
281d1bbd | 270 | pr_debug("kfd: gpu id 0x%x was not found\n", args->gpu_id); |
39b027d9 | 271 | return -EINVAL; |
ff3d04a1 | 272 | } |
39b027d9 OG |
273 | |
274 | mutex_lock(&p->mutex); | |
275 | ||
276 | pdd = kfd_bind_process_to_device(dev, p); | |
66333cb3 | 277 | if (IS_ERR(pdd)) { |
524a6404 | 278 | err = -ESRCH; |
39b027d9 OG |
279 | goto err_bind_process; |
280 | } | |
281 | ||
282 | pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n", | |
283 | p->pasid, | |
284 | dev->id); | |
285 | ||
85dfaef3 BG |
286 | err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, |
287 | 0, q_properties.type, &queue_id); | |
39b027d9 OG |
288 | if (err != 0) |
289 | goto err_create_queue; | |
290 | ||
524a6404 | 291 | args->queue_id = queue_id; |
39b027d9 | 292 | |
f3a39818 | 293 | |
39b027d9 | 294 | /* Return gpu_id as doorbell offset for mmap usage */ |
f3a39818 AL |
295 | args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id); |
296 | args->doorbell_offset <<= PAGE_SHIFT; | |
39b027d9 OG |
297 | |
298 | mutex_unlock(&p->mutex); | |
299 | ||
524a6404 | 300 | pr_debug("kfd: queue id %d was created successfully\n", args->queue_id); |
39b027d9 OG |
301 | |
302 | pr_debug("ring buffer address == 0x%016llX\n", | |
524a6404 | 303 | args->ring_base_address); |
39b027d9 OG |
304 | |
305 | pr_debug("read ptr address == 0x%016llX\n", | |
524a6404 | 306 | args->read_pointer_address); |
39b027d9 OG |
307 | |
308 | pr_debug("write ptr address == 0x%016llX\n", | |
524a6404 | 309 | args->write_pointer_address); |
39b027d9 OG |
310 | |
311 | return 0; | |
312 | ||
39b027d9 OG |
313 | err_create_queue: |
314 | err_bind_process: | |
315 | mutex_unlock(&p->mutex); | |
316 | return err; | |
4a488a7a OG |
317 | } |
318 | ||
319 | static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, | |
524a6404 | 320 | void *data) |
4a488a7a | 321 | { |
39b027d9 | 322 | int retval; |
524a6404 | 323 | struct kfd_ioctl_destroy_queue_args *args = data; |
39b027d9 OG |
324 | |
325 | pr_debug("kfd: destroying queue id %d for PASID %d\n", | |
524a6404 | 326 | args->queue_id, |
39b027d9 OG |
327 | p->pasid); |
328 | ||
329 | mutex_lock(&p->mutex); | |
330 | ||
524a6404 | 331 | retval = pqm_destroy_queue(&p->pqm, args->queue_id); |
39b027d9 OG |
332 | |
333 | mutex_unlock(&p->mutex); | |
334 | return retval; | |
4a488a7a OG |
335 | } |
336 | ||
337 | static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, | |
524a6404 | 338 | void *data) |
4a488a7a | 339 | { |
39b027d9 | 340 | int retval; |
524a6404 | 341 | struct kfd_ioctl_update_queue_args *args = data; |
39b027d9 OG |
342 | struct queue_properties properties; |
343 | ||
524a6404 | 344 | if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { |
39b027d9 OG |
345 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); |
346 | return -EINVAL; | |
347 | } | |
348 | ||
524a6404 | 349 | if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { |
39b027d9 OG |
350 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); |
351 | return -EINVAL; | |
352 | } | |
353 | ||
524a6404 | 354 | if ((args->ring_base_address) && |
4307d8f6 | 355 | (!access_ok(VERIFY_WRITE, |
524a6404 | 356 | (const void __user *) args->ring_base_address, |
4307d8f6 | 357 | sizeof(uint64_t)))) { |
39b027d9 OG |
358 | pr_err("kfd: can't access ring base address\n"); |
359 | return -EFAULT; | |
360 | } | |
361 | ||
524a6404 | 362 | if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { |
39b027d9 OG |
363 | pr_err("kfd: ring size must be a power of 2 or 0\n"); |
364 | return -EINVAL; | |
365 | } | |
366 | ||
524a6404 OG |
367 | properties.queue_address = args->ring_base_address; |
368 | properties.queue_size = args->ring_size; | |
369 | properties.queue_percent = args->queue_percentage; | |
370 | properties.priority = args->queue_priority; | |
39b027d9 OG |
371 | |
372 | pr_debug("kfd: updating queue id %d for PASID %d\n", | |
524a6404 | 373 | args->queue_id, p->pasid); |
39b027d9 OG |
374 | |
375 | mutex_lock(&p->mutex); | |
376 | ||
524a6404 | 377 | retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); |
39b027d9 OG |
378 | |
379 | mutex_unlock(&p->mutex); | |
380 | ||
381 | return retval; | |
4a488a7a OG |
382 | } |
383 | ||
524a6404 OG |
384 | static int kfd_ioctl_set_memory_policy(struct file *filep, |
385 | struct kfd_process *p, void *data) | |
4a488a7a | 386 | { |
524a6404 | 387 | struct kfd_ioctl_set_memory_policy_args *args = data; |
41a286fa AL |
388 | struct kfd_dev *dev; |
389 | int err = 0; | |
390 | struct kfd_process_device *pdd; | |
391 | enum cache_policy default_policy, alternate_policy; | |
392 | ||
524a6404 OG |
393 | if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT |
394 | && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | |
41a286fa AL |
395 | return -EINVAL; |
396 | } | |
397 | ||
524a6404 OG |
398 | if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT |
399 | && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | |
41a286fa AL |
400 | return -EINVAL; |
401 | } | |
402 | ||
524a6404 | 403 | dev = kfd_device_by_id(args->gpu_id); |
41a286fa AL |
404 | if (dev == NULL) |
405 | return -EINVAL; | |
406 | ||
407 | mutex_lock(&p->mutex); | |
408 | ||
409 | pdd = kfd_bind_process_to_device(dev, p); | |
66333cb3 | 410 | if (IS_ERR(pdd)) { |
524a6404 | 411 | err = -ESRCH; |
41a286fa AL |
412 | goto out; |
413 | } | |
414 | ||
524a6404 | 415 | default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) |
41a286fa AL |
416 | ? cache_policy_coherent : cache_policy_noncoherent; |
417 | ||
418 | alternate_policy = | |
524a6404 | 419 | (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) |
41a286fa AL |
420 | ? cache_policy_coherent : cache_policy_noncoherent; |
421 | ||
45c9a5e4 | 422 | if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, |
41a286fa AL |
423 | &pdd->qpd, |
424 | default_policy, | |
425 | alternate_policy, | |
524a6404 OG |
426 | (void __user *)args->alternate_aperture_base, |
427 | args->alternate_aperture_size)) | |
41a286fa AL |
428 | err = -EINVAL; |
429 | ||
430 | out: | |
431 | mutex_unlock(&p->mutex); | |
432 | ||
433 | return err; | |
4a488a7a OG |
434 | } |
435 | ||
aef11009 YS |
436 | static int kfd_ioctl_dbg_register(struct file *filep, |
437 | struct kfd_process *p, void *data) | |
438 | { | |
037ed9a2 YS |
439 | struct kfd_ioctl_dbg_register_args *args = data; |
440 | struct kfd_dev *dev; | |
441 | struct kfd_dbgmgr *dbgmgr_ptr; | |
442 | struct kfd_process_device *pdd; | |
443 | bool create_ok; | |
444 | long status = 0; | |
445 | ||
446 | dev = kfd_device_by_id(args->gpu_id); | |
447 | if (dev == NULL) | |
448 | return -EINVAL; | |
449 | ||
450 | if (dev->device_info->asic_family == CHIP_CARRIZO) { | |
451 | pr_debug("kfd_ioctl_dbg_register not supported on CZ\n"); | |
452 | return -EINVAL; | |
453 | } | |
454 | ||
455 | mutex_lock(kfd_get_dbgmgr_mutex()); | |
456 | mutex_lock(&p->mutex); | |
457 | ||
458 | /* | |
459 | * make sure that we have pdd, if this the first queue created for | |
460 | * this process | |
461 | */ | |
462 | pdd = kfd_bind_process_to_device(dev, p); | |
463 | if (IS_ERR(pdd)) { | |
464 | mutex_unlock(&p->mutex); | |
465 | mutex_unlock(kfd_get_dbgmgr_mutex()); | |
466 | return PTR_ERR(pdd); | |
467 | } | |
468 | ||
469 | if (dev->dbgmgr == NULL) { | |
470 | /* In case of a legal call, we have no dbgmgr yet */ | |
471 | create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev); | |
472 | if (create_ok) { | |
473 | status = kfd_dbgmgr_register(dbgmgr_ptr, p); | |
474 | if (status != 0) | |
475 | kfd_dbgmgr_destroy(dbgmgr_ptr); | |
476 | else | |
477 | dev->dbgmgr = dbgmgr_ptr; | |
478 | } | |
479 | } else { | |
480 | pr_debug("debugger already registered\n"); | |
481 | status = -EINVAL; | |
482 | } | |
483 | ||
484 | mutex_unlock(&p->mutex); | |
485 | mutex_unlock(kfd_get_dbgmgr_mutex()); | |
aef11009 YS |
486 | |
487 | return status; | |
488 | } | |
489 | ||
490 | static int kfd_ioctl_dbg_unrgesiter(struct file *filep, | |
491 | struct kfd_process *p, void *data) | |
492 | { | |
037ed9a2 YS |
493 | struct kfd_ioctl_dbg_unregister_args *args = data; |
494 | struct kfd_dev *dev; | |
495 | long status; | |
496 | ||
497 | dev = kfd_device_by_id(args->gpu_id); | |
498 | if (dev == NULL) | |
499 | return -EINVAL; | |
500 | ||
501 | if (dev->device_info->asic_family == CHIP_CARRIZO) { | |
502 | pr_debug("kfd_ioctl_dbg_unrgesiter not supported on CZ\n"); | |
503 | return -EINVAL; | |
504 | } | |
505 | ||
506 | mutex_lock(kfd_get_dbgmgr_mutex()); | |
507 | ||
508 | status = kfd_dbgmgr_unregister(dev->dbgmgr, p); | |
509 | if (status == 0) { | |
510 | kfd_dbgmgr_destroy(dev->dbgmgr); | |
511 | dev->dbgmgr = NULL; | |
512 | } | |
513 | ||
514 | mutex_unlock(kfd_get_dbgmgr_mutex()); | |
aef11009 YS |
515 | |
516 | return status; | |
517 | } | |
518 | ||
519 | /* | |
520 | * Parse and generate variable size data structure for address watch. | |
521 | * Total size of the buffer and # watch points is limited in order | |
522 | * to prevent kernel abuse. (no bearing to the much smaller HW limitation | |
523 | * which is enforced by dbgdev module) | |
524 | * please also note that the watch address itself are not "copied from user", | |
525 | * since it be set into the HW in user mode values. | |
526 | * | |
527 | */ | |
528 | static int kfd_ioctl_dbg_address_watch(struct file *filep, | |
529 | struct kfd_process *p, void *data) | |
530 | { | |
f8bd1333 YS |
531 | struct kfd_ioctl_dbg_address_watch_args *args = data; |
532 | struct kfd_dev *dev; | |
533 | struct dbg_address_watch_info aw_info; | |
534 | unsigned char *args_buff; | |
535 | long status; | |
536 | void __user *cmd_from_user; | |
537 | uint64_t watch_mask_value = 0; | |
538 | unsigned int args_idx = 0; | |
539 | ||
540 | memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info)); | |
541 | ||
542 | dev = kfd_device_by_id(args->gpu_id); | |
543 | if (dev == NULL) | |
544 | return -EINVAL; | |
545 | ||
546 | if (dev->device_info->asic_family == CHIP_CARRIZO) { | |
547 | pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); | |
548 | return -EINVAL; | |
549 | } | |
550 | ||
551 | cmd_from_user = (void __user *) args->content_ptr; | |
552 | ||
553 | /* Validate arguments */ | |
554 | ||
555 | if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) || | |
7861c7a4 | 556 | (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) || |
f8bd1333 YS |
557 | (cmd_from_user == NULL)) |
558 | return -EINVAL; | |
559 | ||
560 | /* this is the actual buffer to work with */ | |
561 | ||
562 | args_buff = kmalloc(args->buf_size_in_bytes - | |
563 | sizeof(*args), GFP_KERNEL); | |
564 | if (args_buff == NULL) | |
565 | return -ENOMEM; | |
566 | ||
567 | status = copy_from_user(args_buff, cmd_from_user, | |
568 | args->buf_size_in_bytes - sizeof(*args)); | |
569 | ||
570 | if (status != 0) { | |
571 | pr_debug("Failed to copy address watch user data\n"); | |
572 | kfree(args_buff); | |
573 | return -EINVAL; | |
574 | } | |
575 | ||
576 | aw_info.process = p; | |
577 | ||
578 | aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx])); | |
579 | args_idx += sizeof(aw_info.num_watch_points); | |
580 | ||
581 | aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx]; | |
582 | args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points; | |
583 | ||
584 | /* | |
585 | * set watch address base pointer to point on the array base | |
586 | * within args_buff | |
587 | */ | |
588 | aw_info.watch_address = (uint64_t *) &args_buff[args_idx]; | |
589 | ||
590 | /* skip over the addresses buffer */ | |
591 | args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points; | |
592 | ||
7861c7a4 | 593 | if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) { |
f8bd1333 YS |
594 | kfree(args_buff); |
595 | return -EINVAL; | |
596 | } | |
597 | ||
598 | watch_mask_value = (uint64_t) args_buff[args_idx]; | |
599 | ||
600 | if (watch_mask_value > 0) { | |
601 | /* | |
602 | * There is an array of masks. | |
603 | * set watch mask base pointer to point on the array base | |
604 | * within args_buff | |
605 | */ | |
606 | aw_info.watch_mask = (uint64_t *) &args_buff[args_idx]; | |
607 | ||
608 | /* skip over the masks buffer */ | |
609 | args_idx += sizeof(aw_info.watch_mask) * | |
610 | aw_info.num_watch_points; | |
611 | } else { | |
612 | /* just the NULL mask, set to NULL and skip over it */ | |
613 | aw_info.watch_mask = NULL; | |
614 | args_idx += sizeof(aw_info.watch_mask); | |
615 | } | |
616 | ||
7861c7a4 | 617 | if (args_idx >= args->buf_size_in_bytes - sizeof(args)) { |
f8bd1333 YS |
618 | kfree(args_buff); |
619 | return -EINVAL; | |
620 | } | |
621 | ||
622 | /* Currently HSA Event is not supported for DBG */ | |
623 | aw_info.watch_event = NULL; | |
624 | ||
625 | mutex_lock(kfd_get_dbgmgr_mutex()); | |
626 | ||
627 | status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info); | |
628 | ||
629 | mutex_unlock(kfd_get_dbgmgr_mutex()); | |
630 | ||
631 | kfree(args_buff); | |
aef11009 YS |
632 | |
633 | return status; | |
634 | } | |
635 | ||
636 | /* Parse and generate fixed size data structure for wave control */ | |
637 | static int kfd_ioctl_dbg_wave_control(struct file *filep, | |
638 | struct kfd_process *p, void *data) | |
639 | { | |
94484589 YS |
640 | struct kfd_ioctl_dbg_wave_control_args *args = data; |
641 | struct kfd_dev *dev; | |
642 | struct dbg_wave_control_info wac_info; | |
643 | unsigned char *args_buff; | |
644 | uint32_t computed_buff_size; | |
645 | long status; | |
646 | void __user *cmd_from_user; | |
647 | unsigned int args_idx = 0; | |
648 | ||
649 | memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info)); | |
650 | ||
651 | /* we use compact form, independent of the packing attribute value */ | |
652 | computed_buff_size = sizeof(*args) + | |
653 | sizeof(wac_info.mode) + | |
654 | sizeof(wac_info.operand) + | |
655 | sizeof(wac_info.dbgWave_msg.DbgWaveMsg) + | |
656 | sizeof(wac_info.dbgWave_msg.MemoryVA) + | |
657 | sizeof(wac_info.trapId); | |
658 | ||
659 | dev = kfd_device_by_id(args->gpu_id); | |
660 | if (dev == NULL) | |
661 | return -EINVAL; | |
662 | ||
663 | if (dev->device_info->asic_family == CHIP_CARRIZO) { | |
664 | pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); | |
665 | return -EINVAL; | |
666 | } | |
667 | ||
668 | /* input size must match the computed "compact" size */ | |
669 | if (args->buf_size_in_bytes != computed_buff_size) { | |
670 | pr_debug("size mismatch, computed : actual %u : %u\n", | |
671 | args->buf_size_in_bytes, computed_buff_size); | |
672 | return -EINVAL; | |
673 | } | |
674 | ||
675 | cmd_from_user = (void __user *) args->content_ptr; | |
676 | ||
677 | if (cmd_from_user == NULL) | |
678 | return -EINVAL; | |
679 | ||
680 | /* this is the actual buffer to work with */ | |
681 | ||
682 | args_buff = kmalloc(args->buf_size_in_bytes - sizeof(*args), | |
683 | GFP_KERNEL); | |
684 | ||
685 | if (args_buff == NULL) | |
686 | return -ENOMEM; | |
687 | ||
688 | /* Now copy the entire buffer from user */ | |
689 | status = copy_from_user(args_buff, cmd_from_user, | |
690 | args->buf_size_in_bytes - sizeof(*args)); | |
691 | if (status != 0) { | |
692 | pr_debug("Failed to copy wave control user data\n"); | |
693 | kfree(args_buff); | |
694 | return -EINVAL; | |
695 | } | |
696 | ||
697 | /* move ptr to the start of the "pay-load" area */ | |
698 | wac_info.process = p; | |
699 | ||
700 | wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx])); | |
701 | args_idx += sizeof(wac_info.operand); | |
702 | ||
703 | wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx])); | |
704 | args_idx += sizeof(wac_info.mode); | |
705 | ||
706 | wac_info.trapId = *((uint32_t *)(&args_buff[args_idx])); | |
707 | args_idx += sizeof(wac_info.trapId); | |
708 | ||
709 | wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = | |
710 | *((uint32_t *)(&args_buff[args_idx])); | |
711 | wac_info.dbgWave_msg.MemoryVA = NULL; | |
712 | ||
713 | mutex_lock(kfd_get_dbgmgr_mutex()); | |
714 | ||
715 | pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n", | |
716 | wac_info.process, wac_info.operand, | |
717 | wac_info.mode, wac_info.trapId, | |
718 | wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value); | |
719 | ||
720 | status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info); | |
721 | ||
722 | pr_debug("Returned status of dbg manager is %ld\n", status); | |
723 | ||
724 | mutex_unlock(kfd_get_dbgmgr_mutex()); | |
725 | ||
726 | kfree(args_buff); | |
aef11009 YS |
727 | |
728 | return status; | |
729 | } | |
730 | ||
524a6404 OG |
731 | static int kfd_ioctl_get_clock_counters(struct file *filep, |
732 | struct kfd_process *p, void *data) | |
4a488a7a | 733 | { |
524a6404 | 734 | struct kfd_ioctl_get_clock_counters_args *args = data; |
4fac47c8 | 735 | struct kfd_dev *dev; |
affa7d86 | 736 | struct timespec64 time; |
4fac47c8 | 737 | |
524a6404 | 738 | dev = kfd_device_by_id(args->gpu_id); |
4fac47c8 EP |
739 | if (dev == NULL) |
740 | return -EINVAL; | |
741 | ||
742 | /* Reading GPU clock counter from KGD */ | |
cea405b1 XZ |
743 | args->gpu_clock_counter = |
744 | dev->kfd2kgd->get_gpu_clock_counter(dev->kgd); | |
4fac47c8 EP |
745 | |
746 | /* No access to rdtsc. Using raw monotonic time */ | |
affa7d86 JS |
747 | getrawmonotonic64(&time); |
748 | args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time); | |
4fac47c8 | 749 | |
affa7d86 JS |
750 | get_monotonic_boottime64(&time); |
751 | args->system_clock_counter = (uint64_t)timespec64_to_ns(&time); | |
4fac47c8 EP |
752 | |
753 | /* Since the counter is in nano-seconds we use 1GHz frequency */ | |
524a6404 | 754 | args->system_clock_freq = 1000000000; |
4fac47c8 EP |
755 | |
756 | return 0; | |
4a488a7a OG |
757 | } |
758 | ||
759 | ||
760 | static int kfd_ioctl_get_process_apertures(struct file *filp, | |
524a6404 | 761 | struct kfd_process *p, void *data) |
4a488a7a | 762 | { |
524a6404 | 763 | struct kfd_ioctl_get_process_apertures_args *args = data; |
775921ed AS |
764 | struct kfd_process_device_apertures *pAperture; |
765 | struct kfd_process_device *pdd; | |
766 | ||
767 | dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); | |
768 | ||
524a6404 | 769 | args->num_of_nodes = 0; |
775921ed AS |
770 | |
771 | mutex_lock(&p->mutex); | |
772 | ||
773 | /*if the process-device list isn't empty*/ | |
774 | if (kfd_has_process_device_data(p)) { | |
775 | /* Run over all pdd of the process */ | |
776 | pdd = kfd_get_first_process_device_data(p); | |
777 | do { | |
524a6404 OG |
778 | pAperture = |
779 | &args->process_apertures[args->num_of_nodes]; | |
775921ed AS |
780 | pAperture->gpu_id = pdd->dev->id; |
781 | pAperture->lds_base = pdd->lds_base; | |
782 | pAperture->lds_limit = pdd->lds_limit; | |
783 | pAperture->gpuvm_base = pdd->gpuvm_base; | |
784 | pAperture->gpuvm_limit = pdd->gpuvm_limit; | |
785 | pAperture->scratch_base = pdd->scratch_base; | |
786 | pAperture->scratch_limit = pdd->scratch_limit; | |
787 | ||
788 | dev_dbg(kfd_device, | |
524a6404 | 789 | "node id %u\n", args->num_of_nodes); |
775921ed AS |
790 | dev_dbg(kfd_device, |
791 | "gpu id %u\n", pdd->dev->id); | |
792 | dev_dbg(kfd_device, | |
793 | "lds_base %llX\n", pdd->lds_base); | |
794 | dev_dbg(kfd_device, | |
795 | "lds_limit %llX\n", pdd->lds_limit); | |
796 | dev_dbg(kfd_device, | |
797 | "gpuvm_base %llX\n", pdd->gpuvm_base); | |
798 | dev_dbg(kfd_device, | |
799 | "gpuvm_limit %llX\n", pdd->gpuvm_limit); | |
800 | dev_dbg(kfd_device, | |
801 | "scratch_base %llX\n", pdd->scratch_base); | |
802 | dev_dbg(kfd_device, | |
803 | "scratch_limit %llX\n", pdd->scratch_limit); | |
804 | ||
524a6404 | 805 | args->num_of_nodes++; |
775921ed | 806 | } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && |
524a6404 | 807 | (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); |
775921ed AS |
808 | } |
809 | ||
810 | mutex_unlock(&p->mutex); | |
811 | ||
775921ed | 812 | return 0; |
4a488a7a OG |
813 | } |
814 | ||
29a5d3eb AL |
815 | static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, |
816 | void *data) | |
817 | { | |
8377396b AL |
818 | struct kfd_ioctl_create_event_args *args = data; |
819 | int err; | |
820 | ||
821 | err = kfd_event_create(filp, p, args->event_type, | |
822 | args->auto_reset != 0, args->node_id, | |
823 | &args->event_id, &args->event_trigger_data, | |
824 | &args->event_page_offset, | |
825 | &args->event_slot_index); | |
826 | ||
827 | return err; | |
29a5d3eb AL |
828 | } |
829 | ||
830 | static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, | |
831 | void *data) | |
832 | { | |
8377396b AL |
833 | struct kfd_ioctl_destroy_event_args *args = data; |
834 | ||
835 | return kfd_event_destroy(p, args->event_id); | |
29a5d3eb AL |
836 | } |
837 | ||
838 | static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, | |
839 | void *data) | |
840 | { | |
8377396b AL |
841 | struct kfd_ioctl_set_event_args *args = data; |
842 | ||
843 | return kfd_set_event(p, args->event_id); | |
29a5d3eb AL |
844 | } |
845 | ||
846 | static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, | |
847 | void *data) | |
848 | { | |
8377396b AL |
849 | struct kfd_ioctl_reset_event_args *args = data; |
850 | ||
851 | return kfd_reset_event(p, args->event_id); | |
29a5d3eb AL |
852 | } |
853 | ||
854 | static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, | |
855 | void *data) | |
856 | { | |
8377396b AL |
857 | struct kfd_ioctl_wait_events_args *args = data; |
858 | enum kfd_event_wait_result wait_result; | |
859 | int err; | |
860 | ||
861 | err = kfd_wait_on_events(p, args->num_events, | |
862 | (void __user *)args->events_ptr, | |
863 | (args->wait_for_all != 0), | |
864 | args->timeout, &wait_result); | |
865 | ||
866 | args->wait_result = wait_result; | |
867 | ||
868 | return err; | |
29a5d3eb AL |
869 | } |
870 | ||
76baee6c OG |
871 | #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ |
872 | [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl} | |
873 | ||
874 | /** Ioctl table */ | |
875 | static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { | |
876 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, | |
877 | kfd_ioctl_get_version, 0), | |
878 | ||
879 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, | |
880 | kfd_ioctl_create_queue, 0), | |
881 | ||
882 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, | |
883 | kfd_ioctl_destroy_queue, 0), | |
884 | ||
885 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, | |
886 | kfd_ioctl_set_memory_policy, 0), | |
887 | ||
888 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, | |
889 | kfd_ioctl_get_clock_counters, 0), | |
890 | ||
891 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, | |
892 | kfd_ioctl_get_process_apertures, 0), | |
893 | ||
894 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, | |
895 | kfd_ioctl_update_queue, 0), | |
29a5d3eb AL |
896 | |
897 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT, | |
898 | kfd_ioctl_create_event, 0), | |
899 | ||
900 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT, | |
901 | kfd_ioctl_destroy_event, 0), | |
902 | ||
903 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT, | |
904 | kfd_ioctl_set_event, 0), | |
905 | ||
906 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT, | |
907 | kfd_ioctl_reset_event, 0), | |
908 | ||
909 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS, | |
910 | kfd_ioctl_wait_events, 0), | |
aef11009 YS |
911 | |
912 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER, | |
913 | kfd_ioctl_dbg_register, 0), | |
914 | ||
915 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER, | |
916 | kfd_ioctl_dbg_unrgesiter, 0), | |
917 | ||
918 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH, | |
919 | kfd_ioctl_dbg_address_watch, 0), | |
920 | ||
921 | AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL, | |
922 | kfd_ioctl_dbg_wave_control, 0), | |
76baee6c OG |
923 | }; |
924 | ||
925 | #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) | |
926 | ||
4a488a7a OG |
927 | static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) |
928 | { | |
929 | struct kfd_process *process; | |
76baee6c OG |
930 | amdkfd_ioctl_t *func; |
931 | const struct amdkfd_ioctl_desc *ioctl = NULL; | |
932 | unsigned int nr = _IOC_NR(cmd); | |
524a6404 OG |
933 | char stack_kdata[128]; |
934 | char *kdata = NULL; | |
935 | unsigned int usize, asize; | |
936 | int retcode = -EINVAL; | |
4a488a7a | 937 | |
76baee6c OG |
938 | if (nr >= AMDKFD_CORE_IOCTL_COUNT) |
939 | goto err_i1; | |
940 | ||
941 | if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { | |
942 | u32 amdkfd_size; | |
943 | ||
944 | ioctl = &amdkfd_ioctls[nr]; | |
945 | ||
946 | amdkfd_size = _IOC_SIZE(ioctl->cmd); | |
947 | usize = asize = _IOC_SIZE(cmd); | |
948 | if (amdkfd_size > asize) | |
949 | asize = amdkfd_size; | |
950 | ||
951 | cmd = ioctl->cmd; | |
952 | } else | |
953 | goto err_i1; | |
954 | ||
955 | dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); | |
4a488a7a | 956 | |
19f6d2a6 | 957 | process = kfd_get_process(current); |
76baee6c OG |
958 | if (IS_ERR(process)) { |
959 | dev_dbg(kfd_device, "no process\n"); | |
960 | goto err_i1; | |
961 | } | |
4a488a7a | 962 | |
76baee6c OG |
963 | /* Do not trust userspace, use our own definition */ |
964 | func = ioctl->func; | |
965 | ||
966 | if (unlikely(!func)) { | |
967 | dev_dbg(kfd_device, "no function\n"); | |
968 | retcode = -EINVAL; | |
969 | goto err_i1; | |
4a488a7a OG |
970 | } |
971 | ||
524a6404 OG |
972 | if (cmd & (IOC_IN | IOC_OUT)) { |
973 | if (asize <= sizeof(stack_kdata)) { | |
974 | kdata = stack_kdata; | |
975 | } else { | |
976 | kdata = kmalloc(asize, GFP_KERNEL); | |
977 | if (!kdata) { | |
978 | retcode = -ENOMEM; | |
979 | goto err_i1; | |
980 | } | |
981 | } | |
982 | if (asize > usize) | |
983 | memset(kdata + usize, 0, asize - usize); | |
984 | } | |
4a488a7a | 985 | |
524a6404 OG |
986 | if (cmd & IOC_IN) { |
987 | if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { | |
988 | retcode = -EFAULT; | |
989 | goto err_i1; | |
990 | } | |
991 | } else if (cmd & IOC_OUT) { | |
992 | memset(kdata, 0, usize); | |
993 | } | |
994 | ||
76baee6c | 995 | retcode = func(filep, process, kdata); |
4a488a7a | 996 | |
524a6404 OG |
997 | if (cmd & IOC_OUT) |
998 | if (copy_to_user((void __user *)arg, kdata, usize) != 0) | |
999 | retcode = -EFAULT; | |
4a488a7a | 1000 | |
524a6404 | 1001 | err_i1: |
76baee6c OG |
1002 | if (!ioctl) |
1003 | dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", | |
1004 | task_pid_nr(current), cmd, nr); | |
1005 | ||
524a6404 OG |
1006 | if (kdata != stack_kdata) |
1007 | kfree(kdata); | |
1008 | ||
1009 | if (retcode) | |
1010 | dev_dbg(kfd_device, "ret = %d\n", retcode); | |
1011 | ||
1012 | return retcode; | |
4a488a7a | 1013 | } |
19f6d2a6 OG |
1014 | |
1015 | static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) | |
1016 | { | |
1017 | struct kfd_process *process; | |
1018 | ||
1019 | process = kfd_get_process(current); | |
1020 | if (IS_ERR(process)) | |
1021 | return PTR_ERR(process); | |
1022 | ||
f3a39818 AL |
1023 | if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) == |
1024 | KFD_MMAP_DOORBELL_MASK) { | |
1025 | vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK; | |
1026 | return kfd_doorbell_mmap(process, vma); | |
1027 | } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) == | |
1028 | KFD_MMAP_EVENTS_MASK) { | |
1029 | vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK; | |
1030 | return kfd_event_mmap(process, vma); | |
1031 | } | |
1032 | ||
1033 | return -EFAULT; | |
19f6d2a6 | 1034 | } |