2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/uaccess.h>
30 #include <linux/compat.h>
31 #include <uapi/linux/kfd_ioctl.h>
32 #include <linux/time.h>
34 #include <linux/mman.h>
35 #include <asm/processor.h>
37 #include "kfd_device_queue_manager.h"
38 #include "kfd_dbgmgr.h"
40 static long kfd_ioctl(struct file
*, unsigned int, unsigned long);
41 static int kfd_open(struct inode
*, struct file
*);
42 static int kfd_mmap(struct file
*, struct vm_area_struct
*);
44 static const char kfd_dev_name
[] = "kfd";
46 static const struct file_operations kfd_fops
= {
48 .unlocked_ioctl
= kfd_ioctl
,
49 .compat_ioctl
= kfd_ioctl
,
54 static int kfd_char_dev_major
= -1;
55 static struct class *kfd_class
;
56 struct device
*kfd_device
;
58 int kfd_chardev_init(void)
62 kfd_char_dev_major
= register_chrdev(0, kfd_dev_name
, &kfd_fops
);
63 err
= kfd_char_dev_major
;
65 goto err_register_chrdev
;
67 kfd_class
= class_create(THIS_MODULE
, kfd_dev_name
);
68 err
= PTR_ERR(kfd_class
);
69 if (IS_ERR(kfd_class
))
70 goto err_class_create
;
72 kfd_device
= device_create(kfd_class
, NULL
,
73 MKDEV(kfd_char_dev_major
, 0),
75 err
= PTR_ERR(kfd_device
);
76 if (IS_ERR(kfd_device
))
77 goto err_device_create
;
82 class_destroy(kfd_class
);
84 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
89 void kfd_chardev_exit(void)
91 device_destroy(kfd_class
, MKDEV(kfd_char_dev_major
, 0));
92 class_destroy(kfd_class
);
93 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
96 struct device
*kfd_chardev(void)
102 static int kfd_open(struct inode
*inode
, struct file
*filep
)
104 struct kfd_process
*process
;
105 bool is_32bit_user_mode
;
107 if (iminor(inode
) != 0)
110 is_32bit_user_mode
= is_compat_task();
112 if (is_32bit_user_mode
== true) {
114 "Process %d (32-bit) failed to open /dev/kfd\n"
115 "32-bit processes are not supported by amdkfd\n",
120 process
= kfd_create_process(current
);
122 return PTR_ERR(process
);
124 dev_dbg(kfd_device
, "process %d opened, compat mode (32 bit) - %d\n",
125 process
->pasid
, process
->is_32bit_user_mode
);
130 static int kfd_ioctl_get_version(struct file
*filep
, struct kfd_process
*p
,
133 struct kfd_ioctl_get_version_args
*args
= data
;
136 args
->major_version
= KFD_IOCTL_MAJOR_VERSION
;
137 args
->minor_version
= KFD_IOCTL_MINOR_VERSION
;
142 static int set_queue_properties_from_user(struct queue_properties
*q_properties
,
143 struct kfd_ioctl_create_queue_args
*args
)
145 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
146 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
150 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
151 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
155 if ((args
->ring_base_address
) &&
156 (!access_ok(VERIFY_WRITE
,
157 (const void __user
*) args
->ring_base_address
,
158 sizeof(uint64_t)))) {
159 pr_err("kfd: can't access ring base address\n");
163 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
164 pr_err("kfd: ring size must be a power of 2 or 0\n");
168 if (!access_ok(VERIFY_WRITE
,
169 (const void __user
*) args
->read_pointer_address
,
171 pr_err("kfd: can't access read pointer\n");
175 if (!access_ok(VERIFY_WRITE
,
176 (const void __user
*) args
->write_pointer_address
,
178 pr_err("kfd: can't access write pointer\n");
182 if (args
->eop_buffer_address
&&
183 !access_ok(VERIFY_WRITE
,
184 (const void __user
*) args
->eop_buffer_address
,
186 pr_debug("kfd: can't access eop buffer");
190 if (args
->ctx_save_restore_address
&&
191 !access_ok(VERIFY_WRITE
,
192 (const void __user
*) args
->ctx_save_restore_address
,
194 pr_debug("kfd: can't access ctx save restore buffer");
198 q_properties
->is_interop
= false;
199 q_properties
->queue_percent
= args
->queue_percentage
;
200 q_properties
->priority
= args
->queue_priority
;
201 q_properties
->queue_address
= args
->ring_base_address
;
202 q_properties
->queue_size
= args
->ring_size
;
203 q_properties
->read_ptr
= (uint32_t *) args
->read_pointer_address
;
204 q_properties
->write_ptr
= (uint32_t *) args
->write_pointer_address
;
205 q_properties
->eop_ring_buffer_address
= args
->eop_buffer_address
;
206 q_properties
->eop_ring_buffer_size
= args
->eop_buffer_size
;
207 q_properties
->ctx_save_restore_area_address
=
208 args
->ctx_save_restore_address
;
209 q_properties
->ctx_save_restore_area_size
= args
->ctx_save_restore_size
;
210 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE
||
211 args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
212 q_properties
->type
= KFD_QUEUE_TYPE_COMPUTE
;
213 else if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_SDMA
)
214 q_properties
->type
= KFD_QUEUE_TYPE_SDMA
;
218 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
219 q_properties
->format
= KFD_QUEUE_FORMAT_AQL
;
221 q_properties
->format
= KFD_QUEUE_FORMAT_PM4
;
223 pr_debug("Queue Percentage (%d, %d)\n",
224 q_properties
->queue_percent
, args
->queue_percentage
);
226 pr_debug("Queue Priority (%d, %d)\n",
227 q_properties
->priority
, args
->queue_priority
);
229 pr_debug("Queue Address (0x%llX, 0x%llX)\n",
230 q_properties
->queue_address
, args
->ring_base_address
);
232 pr_debug("Queue Size (0x%llX, %u)\n",
233 q_properties
->queue_size
, args
->ring_size
);
235 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
236 (uint64_t) q_properties
->read_ptr
,
237 (uint64_t) q_properties
->write_ptr
);
239 pr_debug("Queue Format (%d)\n", q_properties
->format
);
241 pr_debug("Queue EOP (0x%llX)\n", q_properties
->eop_ring_buffer_address
);
243 pr_debug("Queue CTX save arex (0x%llX)\n",
244 q_properties
->ctx_save_restore_area_address
);
249 static int kfd_ioctl_create_queue(struct file
*filep
, struct kfd_process
*p
,
252 struct kfd_ioctl_create_queue_args
*args
= data
;
255 unsigned int queue_id
;
256 struct kfd_process_device
*pdd
;
257 struct queue_properties q_properties
;
259 memset(&q_properties
, 0, sizeof(struct queue_properties
));
261 pr_debug("kfd: creating queue ioctl\n");
263 err
= set_queue_properties_from_user(&q_properties
, args
);
267 pr_debug("kfd: looking for gpu id 0x%x\n", args
->gpu_id
);
268 dev
= kfd_device_by_id(args
->gpu_id
);
270 pr_debug("kfd: gpu id 0x%x was not found\n", args
->gpu_id
);
274 mutex_lock(&p
->mutex
);
276 pdd
= kfd_bind_process_to_device(dev
, p
);
279 goto err_bind_process
;
282 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
286 err
= pqm_create_queue(&p
->pqm
, dev
, filep
, &q_properties
,
287 0, q_properties
.type
, &queue_id
);
289 goto err_create_queue
;
291 args
->queue_id
= queue_id
;
294 /* Return gpu_id as doorbell offset for mmap usage */
295 args
->doorbell_offset
= (KFD_MMAP_DOORBELL_MASK
| args
->gpu_id
);
296 args
->doorbell_offset
<<= PAGE_SHIFT
;
298 mutex_unlock(&p
->mutex
);
300 pr_debug("kfd: queue id %d was created successfully\n", args
->queue_id
);
302 pr_debug("ring buffer address == 0x%016llX\n",
303 args
->ring_base_address
);
305 pr_debug("read ptr address == 0x%016llX\n",
306 args
->read_pointer_address
);
308 pr_debug("write ptr address == 0x%016llX\n",
309 args
->write_pointer_address
);
315 mutex_unlock(&p
->mutex
);
319 static int kfd_ioctl_destroy_queue(struct file
*filp
, struct kfd_process
*p
,
323 struct kfd_ioctl_destroy_queue_args
*args
= data
;
325 pr_debug("kfd: destroying queue id %d for PASID %d\n",
329 mutex_lock(&p
->mutex
);
331 retval
= pqm_destroy_queue(&p
->pqm
, args
->queue_id
);
333 mutex_unlock(&p
->mutex
);
337 static int kfd_ioctl_update_queue(struct file
*filp
, struct kfd_process
*p
,
341 struct kfd_ioctl_update_queue_args
*args
= data
;
342 struct queue_properties properties
;
344 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
345 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
349 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
350 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
354 if ((args
->ring_base_address
) &&
355 (!access_ok(VERIFY_WRITE
,
356 (const void __user
*) args
->ring_base_address
,
357 sizeof(uint64_t)))) {
358 pr_err("kfd: can't access ring base address\n");
362 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
363 pr_err("kfd: ring size must be a power of 2 or 0\n");
367 properties
.queue_address
= args
->ring_base_address
;
368 properties
.queue_size
= args
->ring_size
;
369 properties
.queue_percent
= args
->queue_percentage
;
370 properties
.priority
= args
->queue_priority
;
372 pr_debug("kfd: updating queue id %d for PASID %d\n",
373 args
->queue_id
, p
->pasid
);
375 mutex_lock(&p
->mutex
);
377 retval
= pqm_update_queue(&p
->pqm
, args
->queue_id
, &properties
);
379 mutex_unlock(&p
->mutex
);
384 static int kfd_ioctl_set_memory_policy(struct file
*filep
,
385 struct kfd_process
*p
, void *data
)
387 struct kfd_ioctl_set_memory_policy_args
*args
= data
;
390 struct kfd_process_device
*pdd
;
391 enum cache_policy default_policy
, alternate_policy
;
393 if (args
->default_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
394 && args
->default_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
398 if (args
->alternate_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
399 && args
->alternate_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
403 dev
= kfd_device_by_id(args
->gpu_id
);
407 mutex_lock(&p
->mutex
);
409 pdd
= kfd_bind_process_to_device(dev
, p
);
415 default_policy
= (args
->default_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
416 ? cache_policy_coherent
: cache_policy_noncoherent
;
419 (args
->alternate_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
420 ? cache_policy_coherent
: cache_policy_noncoherent
;
422 if (!dev
->dqm
->ops
.set_cache_memory_policy(dev
->dqm
,
426 (void __user
*)args
->alternate_aperture_base
,
427 args
->alternate_aperture_size
))
431 mutex_unlock(&p
->mutex
);
436 static int kfd_ioctl_dbg_register(struct file
*filep
,
437 struct kfd_process
*p
, void *data
)
439 struct kfd_ioctl_dbg_register_args
*args
= data
;
441 struct kfd_dbgmgr
*dbgmgr_ptr
;
442 struct kfd_process_device
*pdd
;
446 dev
= kfd_device_by_id(args
->gpu_id
);
450 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
451 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
455 mutex_lock(kfd_get_dbgmgr_mutex());
456 mutex_lock(&p
->mutex
);
459 * make sure that we have pdd, if this the first queue created for
462 pdd
= kfd_bind_process_to_device(dev
, p
);
464 mutex_unlock(&p
->mutex
);
465 mutex_unlock(kfd_get_dbgmgr_mutex());
469 if (dev
->dbgmgr
== NULL
) {
470 /* In case of a legal call, we have no dbgmgr yet */
471 create_ok
= kfd_dbgmgr_create(&dbgmgr_ptr
, dev
);
473 status
= kfd_dbgmgr_register(dbgmgr_ptr
, p
);
475 kfd_dbgmgr_destroy(dbgmgr_ptr
);
477 dev
->dbgmgr
= dbgmgr_ptr
;
480 pr_debug("debugger already registered\n");
484 mutex_unlock(&p
->mutex
);
485 mutex_unlock(kfd_get_dbgmgr_mutex());
490 static int kfd_ioctl_dbg_unrgesiter(struct file
*filep
,
491 struct kfd_process
*p
, void *data
)
493 struct kfd_ioctl_dbg_unregister_args
*args
= data
;
497 dev
= kfd_device_by_id(args
->gpu_id
);
501 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
502 pr_debug("kfd_ioctl_dbg_unrgesiter not supported on CZ\n");
506 mutex_lock(kfd_get_dbgmgr_mutex());
508 status
= kfd_dbgmgr_unregister(dev
->dbgmgr
, p
);
510 kfd_dbgmgr_destroy(dev
->dbgmgr
);
514 mutex_unlock(kfd_get_dbgmgr_mutex());
520 * Parse and generate variable size data structure for address watch.
521 * Total size of the buffer and # watch points is limited in order
522 * to prevent kernel abuse. (no bearing to the much smaller HW limitation
523 * which is enforced by dbgdev module)
524 * please also note that the watch address itself are not "copied from user",
525 * since it be set into the HW in user mode values.
528 static int kfd_ioctl_dbg_address_watch(struct file
*filep
,
529 struct kfd_process
*p
, void *data
)
531 struct kfd_ioctl_dbg_address_watch_args
*args
= data
;
533 struct dbg_address_watch_info aw_info
;
534 unsigned char *args_buff
;
536 void __user
*cmd_from_user
;
537 uint64_t watch_mask_value
= 0;
538 unsigned int args_idx
= 0;
540 memset((void *) &aw_info
, 0, sizeof(struct dbg_address_watch_info
));
542 dev
= kfd_device_by_id(args
->gpu_id
);
546 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
547 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
551 cmd_from_user
= (void __user
*) args
->content_ptr
;
553 /* Validate arguments */
555 if ((args
->buf_size_in_bytes
> MAX_ALLOWED_AW_BUFF_SIZE
) ||
556 (args
->buf_size_in_bytes
<= sizeof(*args
) + sizeof(int) * 2) ||
557 (cmd_from_user
== NULL
))
560 /* this is the actual buffer to work with */
561 args_buff
= memdup_user(cmd_from_user
,
562 args
->buf_size_in_bytes
- sizeof(*args
));
563 if (IS_ERR(args_buff
))
564 return PTR_ERR(args_buff
);
568 aw_info
.num_watch_points
= *((uint32_t *)(&args_buff
[args_idx
]));
569 args_idx
+= sizeof(aw_info
.num_watch_points
);
571 aw_info
.watch_mode
= (enum HSA_DBG_WATCH_MODE
*) &args_buff
[args_idx
];
572 args_idx
+= sizeof(enum HSA_DBG_WATCH_MODE
) * aw_info
.num_watch_points
;
575 * set watch address base pointer to point on the array base
578 aw_info
.watch_address
= (uint64_t *) &args_buff
[args_idx
];
580 /* skip over the addresses buffer */
581 args_idx
+= sizeof(aw_info
.watch_address
) * aw_info
.num_watch_points
;
583 if (args_idx
>= args
->buf_size_in_bytes
- sizeof(*args
)) {
588 watch_mask_value
= (uint64_t) args_buff
[args_idx
];
590 if (watch_mask_value
> 0) {
592 * There is an array of masks.
593 * set watch mask base pointer to point on the array base
596 aw_info
.watch_mask
= (uint64_t *) &args_buff
[args_idx
];
598 /* skip over the masks buffer */
599 args_idx
+= sizeof(aw_info
.watch_mask
) *
600 aw_info
.num_watch_points
;
602 /* just the NULL mask, set to NULL and skip over it */
603 aw_info
.watch_mask
= NULL
;
604 args_idx
+= sizeof(aw_info
.watch_mask
);
607 if (args_idx
>= args
->buf_size_in_bytes
- sizeof(args
)) {
612 /* Currently HSA Event is not supported for DBG */
613 aw_info
.watch_event
= NULL
;
615 mutex_lock(kfd_get_dbgmgr_mutex());
617 status
= kfd_dbgmgr_address_watch(dev
->dbgmgr
, &aw_info
);
619 mutex_unlock(kfd_get_dbgmgr_mutex());
626 /* Parse and generate fixed size data structure for wave control */
627 static int kfd_ioctl_dbg_wave_control(struct file
*filep
,
628 struct kfd_process
*p
, void *data
)
630 struct kfd_ioctl_dbg_wave_control_args
*args
= data
;
632 struct dbg_wave_control_info wac_info
;
633 unsigned char *args_buff
;
634 uint32_t computed_buff_size
;
636 void __user
*cmd_from_user
;
637 unsigned int args_idx
= 0;
639 memset((void *) &wac_info
, 0, sizeof(struct dbg_wave_control_info
));
641 /* we use compact form, independent of the packing attribute value */
642 computed_buff_size
= sizeof(*args
) +
643 sizeof(wac_info
.mode
) +
644 sizeof(wac_info
.operand
) +
645 sizeof(wac_info
.dbgWave_msg
.DbgWaveMsg
) +
646 sizeof(wac_info
.dbgWave_msg
.MemoryVA
) +
647 sizeof(wac_info
.trapId
);
649 dev
= kfd_device_by_id(args
->gpu_id
);
653 if (dev
->device_info
->asic_family
== CHIP_CARRIZO
) {
654 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
658 /* input size must match the computed "compact" size */
659 if (args
->buf_size_in_bytes
!= computed_buff_size
) {
660 pr_debug("size mismatch, computed : actual %u : %u\n",
661 args
->buf_size_in_bytes
, computed_buff_size
);
665 cmd_from_user
= (void __user
*) args
->content_ptr
;
667 if (cmd_from_user
== NULL
)
670 /* copy the entire buffer from user */
672 args_buff
= memdup_user(cmd_from_user
,
673 args
->buf_size_in_bytes
- sizeof(*args
));
674 if (IS_ERR(args_buff
))
675 return PTR_ERR(args_buff
);
677 /* move ptr to the start of the "pay-load" area */
678 wac_info
.process
= p
;
680 wac_info
.operand
= *((enum HSA_DBG_WAVEOP
*)(&args_buff
[args_idx
]));
681 args_idx
+= sizeof(wac_info
.operand
);
683 wac_info
.mode
= *((enum HSA_DBG_WAVEMODE
*)(&args_buff
[args_idx
]));
684 args_idx
+= sizeof(wac_info
.mode
);
686 wac_info
.trapId
= *((uint32_t *)(&args_buff
[args_idx
]));
687 args_idx
+= sizeof(wac_info
.trapId
);
689 wac_info
.dbgWave_msg
.DbgWaveMsg
.WaveMsgInfoGen2
.Value
=
690 *((uint32_t *)(&args_buff
[args_idx
]));
691 wac_info
.dbgWave_msg
.MemoryVA
= NULL
;
693 mutex_lock(kfd_get_dbgmgr_mutex());
695 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
696 wac_info
.process
, wac_info
.operand
,
697 wac_info
.mode
, wac_info
.trapId
,
698 wac_info
.dbgWave_msg
.DbgWaveMsg
.WaveMsgInfoGen2
.Value
);
700 status
= kfd_dbgmgr_wave_control(dev
->dbgmgr
, &wac_info
);
702 pr_debug("Returned status of dbg manager is %ld\n", status
);
704 mutex_unlock(kfd_get_dbgmgr_mutex());
711 static int kfd_ioctl_get_clock_counters(struct file
*filep
,
712 struct kfd_process
*p
, void *data
)
714 struct kfd_ioctl_get_clock_counters_args
*args
= data
;
716 struct timespec64 time
;
718 dev
= kfd_device_by_id(args
->gpu_id
);
722 /* Reading GPU clock counter from KGD */
723 args
->gpu_clock_counter
=
724 dev
->kfd2kgd
->get_gpu_clock_counter(dev
->kgd
);
726 /* No access to rdtsc. Using raw monotonic time */
727 getrawmonotonic64(&time
);
728 args
->cpu_clock_counter
= (uint64_t)timespec64_to_ns(&time
);
730 get_monotonic_boottime64(&time
);
731 args
->system_clock_counter
= (uint64_t)timespec64_to_ns(&time
);
733 /* Since the counter is in nano-seconds we use 1GHz frequency */
734 args
->system_clock_freq
= 1000000000;
740 static int kfd_ioctl_get_process_apertures(struct file
*filp
,
741 struct kfd_process
*p
, void *data
)
743 struct kfd_ioctl_get_process_apertures_args
*args
= data
;
744 struct kfd_process_device_apertures
*pAperture
;
745 struct kfd_process_device
*pdd
;
747 dev_dbg(kfd_device
, "get apertures for PASID %d", p
->pasid
);
749 args
->num_of_nodes
= 0;
751 mutex_lock(&p
->mutex
);
753 /*if the process-device list isn't empty*/
754 if (kfd_has_process_device_data(p
)) {
755 /* Run over all pdd of the process */
756 pdd
= kfd_get_first_process_device_data(p
);
759 &args
->process_apertures
[args
->num_of_nodes
];
760 pAperture
->gpu_id
= pdd
->dev
->id
;
761 pAperture
->lds_base
= pdd
->lds_base
;
762 pAperture
->lds_limit
= pdd
->lds_limit
;
763 pAperture
->gpuvm_base
= pdd
->gpuvm_base
;
764 pAperture
->gpuvm_limit
= pdd
->gpuvm_limit
;
765 pAperture
->scratch_base
= pdd
->scratch_base
;
766 pAperture
->scratch_limit
= pdd
->scratch_limit
;
769 "node id %u\n", args
->num_of_nodes
);
771 "gpu id %u\n", pdd
->dev
->id
);
773 "lds_base %llX\n", pdd
->lds_base
);
775 "lds_limit %llX\n", pdd
->lds_limit
);
777 "gpuvm_base %llX\n", pdd
->gpuvm_base
);
779 "gpuvm_limit %llX\n", pdd
->gpuvm_limit
);
781 "scratch_base %llX\n", pdd
->scratch_base
);
783 "scratch_limit %llX\n", pdd
->scratch_limit
);
785 args
->num_of_nodes
++;
786 } while ((pdd
= kfd_get_next_process_device_data(p
, pdd
)) != NULL
&&
787 (args
->num_of_nodes
< NUM_OF_SUPPORTED_GPUS
));
790 mutex_unlock(&p
->mutex
);
795 static int kfd_ioctl_create_event(struct file
*filp
, struct kfd_process
*p
,
798 struct kfd_ioctl_create_event_args
*args
= data
;
801 err
= kfd_event_create(filp
, p
, args
->event_type
,
802 args
->auto_reset
!= 0, args
->node_id
,
803 &args
->event_id
, &args
->event_trigger_data
,
804 &args
->event_page_offset
,
805 &args
->event_slot_index
);
810 static int kfd_ioctl_destroy_event(struct file
*filp
, struct kfd_process
*p
,
813 struct kfd_ioctl_destroy_event_args
*args
= data
;
815 return kfd_event_destroy(p
, args
->event_id
);
818 static int kfd_ioctl_set_event(struct file
*filp
, struct kfd_process
*p
,
821 struct kfd_ioctl_set_event_args
*args
= data
;
823 return kfd_set_event(p
, args
->event_id
);
826 static int kfd_ioctl_reset_event(struct file
*filp
, struct kfd_process
*p
,
829 struct kfd_ioctl_reset_event_args
*args
= data
;
831 return kfd_reset_event(p
, args
->event_id
);
834 static int kfd_ioctl_wait_events(struct file
*filp
, struct kfd_process
*p
,
837 struct kfd_ioctl_wait_events_args
*args
= data
;
838 enum kfd_event_wait_result wait_result
;
841 err
= kfd_wait_on_events(p
, args
->num_events
,
842 (void __user
*)args
->events_ptr
,
843 (args
->wait_for_all
!= 0),
844 args
->timeout
, &wait_result
);
846 args
->wait_result
= wait_result
;
851 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
852 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
855 static const struct amdkfd_ioctl_desc amdkfd_ioctls
[] = {
856 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION
,
857 kfd_ioctl_get_version
, 0),
859 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE
,
860 kfd_ioctl_create_queue
, 0),
862 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE
,
863 kfd_ioctl_destroy_queue
, 0),
865 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY
,
866 kfd_ioctl_set_memory_policy
, 0),
868 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS
,
869 kfd_ioctl_get_clock_counters
, 0),
871 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES
,
872 kfd_ioctl_get_process_apertures
, 0),
874 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE
,
875 kfd_ioctl_update_queue
, 0),
877 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT
,
878 kfd_ioctl_create_event
, 0),
880 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT
,
881 kfd_ioctl_destroy_event
, 0),
883 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT
,
884 kfd_ioctl_set_event
, 0),
886 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT
,
887 kfd_ioctl_reset_event
, 0),
889 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS
,
890 kfd_ioctl_wait_events
, 0),
892 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER
,
893 kfd_ioctl_dbg_register
, 0),
895 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER
,
896 kfd_ioctl_dbg_unrgesiter
, 0),
898 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH
,
899 kfd_ioctl_dbg_address_watch
, 0),
901 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL
,
902 kfd_ioctl_dbg_wave_control
, 0),
905 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
907 static long kfd_ioctl(struct file
*filep
, unsigned int cmd
, unsigned long arg
)
909 struct kfd_process
*process
;
910 amdkfd_ioctl_t
*func
;
911 const struct amdkfd_ioctl_desc
*ioctl
= NULL
;
912 unsigned int nr
= _IOC_NR(cmd
);
913 char stack_kdata
[128];
915 unsigned int usize
, asize
;
916 int retcode
= -EINVAL
;
918 if (nr
>= AMDKFD_CORE_IOCTL_COUNT
)
921 if ((nr
>= AMDKFD_COMMAND_START
) && (nr
< AMDKFD_COMMAND_END
)) {
924 ioctl
= &amdkfd_ioctls
[nr
];
926 amdkfd_size
= _IOC_SIZE(ioctl
->cmd
);
927 usize
= asize
= _IOC_SIZE(cmd
);
928 if (amdkfd_size
> asize
)
935 dev_dbg(kfd_device
, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd
, nr
, arg
);
937 process
= kfd_get_process(current
);
938 if (IS_ERR(process
)) {
939 dev_dbg(kfd_device
, "no process\n");
943 /* Do not trust userspace, use our own definition */
946 if (unlikely(!func
)) {
947 dev_dbg(kfd_device
, "no function\n");
952 if (cmd
& (IOC_IN
| IOC_OUT
)) {
953 if (asize
<= sizeof(stack_kdata
)) {
956 kdata
= kmalloc(asize
, GFP_KERNEL
);
963 memset(kdata
+ usize
, 0, asize
- usize
);
967 if (copy_from_user(kdata
, (void __user
*)arg
, usize
) != 0) {
971 } else if (cmd
& IOC_OUT
) {
972 memset(kdata
, 0, usize
);
975 retcode
= func(filep
, process
, kdata
);
978 if (copy_to_user((void __user
*)arg
, kdata
, usize
) != 0)
983 dev_dbg(kfd_device
, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
984 task_pid_nr(current
), cmd
, nr
);
986 if (kdata
!= stack_kdata
)
990 dev_dbg(kfd_device
, "ret = %d\n", retcode
);
995 static int kfd_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
997 struct kfd_process
*process
;
999 process
= kfd_get_process(current
);
1000 if (IS_ERR(process
))
1001 return PTR_ERR(process
);
1003 if ((vma
->vm_pgoff
& KFD_MMAP_DOORBELL_MASK
) ==
1004 KFD_MMAP_DOORBELL_MASK
) {
1005 vma
->vm_pgoff
= vma
->vm_pgoff
^ KFD_MMAP_DOORBELL_MASK
;
1006 return kfd_doorbell_mmap(process
, vma
);
1007 } else if ((vma
->vm_pgoff
& KFD_MMAP_EVENTS_MASK
) ==
1008 KFD_MMAP_EVENTS_MASK
) {
1009 vma
->vm_pgoff
= vma
->vm_pgoff
^ KFD_MMAP_EVENTS_MASK
;
1010 return kfd_event_mmap(process
, vma
);