2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/uaccess.h>
30 #include <linux/compat.h>
31 #include <uapi/linux/kfd_ioctl.h>
32 #include <linux/time.h>
34 #include <linux/uaccess.h>
35 #include <uapi/asm-generic/mman-common.h>
36 #include <asm/processor.h>
38 #include "kfd_device_queue_manager.h"
40 static long kfd_ioctl(struct file
*, unsigned int, unsigned long);
41 static int kfd_open(struct inode
*, struct file
*);
42 static int kfd_mmap(struct file
*, struct vm_area_struct
*);
44 static const char kfd_dev_name
[] = "kfd";
46 static const struct file_operations kfd_fops
= {
48 .unlocked_ioctl
= kfd_ioctl
,
49 .compat_ioctl
= kfd_ioctl
,
54 static int kfd_char_dev_major
= -1;
55 static struct class *kfd_class
;
56 struct device
*kfd_device
;
58 int kfd_chardev_init(void)
62 kfd_char_dev_major
= register_chrdev(0, kfd_dev_name
, &kfd_fops
);
63 err
= kfd_char_dev_major
;
65 goto err_register_chrdev
;
67 kfd_class
= class_create(THIS_MODULE
, kfd_dev_name
);
68 err
= PTR_ERR(kfd_class
);
69 if (IS_ERR(kfd_class
))
70 goto err_class_create
;
72 kfd_device
= device_create(kfd_class
, NULL
,
73 MKDEV(kfd_char_dev_major
, 0),
75 err
= PTR_ERR(kfd_device
);
76 if (IS_ERR(kfd_device
))
77 goto err_device_create
;
82 class_destroy(kfd_class
);
84 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
89 void kfd_chardev_exit(void)
91 device_destroy(kfd_class
, MKDEV(kfd_char_dev_major
, 0));
92 class_destroy(kfd_class
);
93 unregister_chrdev(kfd_char_dev_major
, kfd_dev_name
);
96 struct device
*kfd_chardev(void)
102 static int kfd_open(struct inode
*inode
, struct file
*filep
)
104 struct kfd_process
*process
;
106 if (iminor(inode
) != 0)
109 process
= kfd_create_process(current
);
111 return PTR_ERR(process
);
113 process
->is_32bit_user_mode
= is_compat_task();
115 dev_dbg(kfd_device
, "process %d opened, compat mode (32 bit) - %d\n",
116 process
->pasid
, process
->is_32bit_user_mode
);
118 kfd_init_apertures(process
);
123 static long kfd_ioctl_get_version(struct file
*filep
, struct kfd_process
*p
,
129 static int set_queue_properties_from_user(struct queue_properties
*q_properties
,
130 struct kfd_ioctl_create_queue_args
*args
)
132 if (args
->queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
133 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
137 if (args
->queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
138 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
142 if ((args
->ring_base_address
) &&
143 (!access_ok(VERIFY_WRITE
, args
->ring_base_address
, sizeof(uint64_t)))) {
144 pr_err("kfd: can't access ring base address\n");
148 if (!is_power_of_2(args
->ring_size
) && (args
->ring_size
!= 0)) {
149 pr_err("kfd: ring size must be a power of 2 or 0\n");
153 if (!access_ok(VERIFY_WRITE
, args
->read_pointer_address
, sizeof(uint32_t))) {
154 pr_err("kfd: can't access read pointer\n");
158 if (!access_ok(VERIFY_WRITE
, args
->write_pointer_address
, sizeof(uint32_t))) {
159 pr_err("kfd: can't access write pointer\n");
163 q_properties
->is_interop
= false;
164 q_properties
->queue_percent
= args
->queue_percentage
;
165 q_properties
->priority
= args
->queue_priority
;
166 q_properties
->queue_address
= args
->ring_base_address
;
167 q_properties
->queue_size
= args
->ring_size
;
168 q_properties
->read_ptr
= (uint32_t *) args
->read_pointer_address
;
169 q_properties
->write_ptr
= (uint32_t *) args
->write_pointer_address
;
170 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE
||
171 args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
172 q_properties
->type
= KFD_QUEUE_TYPE_COMPUTE
;
176 if (args
->queue_type
== KFD_IOC_QUEUE_TYPE_COMPUTE_AQL
)
177 q_properties
->format
= KFD_QUEUE_FORMAT_AQL
;
179 q_properties
->format
= KFD_QUEUE_FORMAT_PM4
;
181 pr_debug("Queue Percentage (%d, %d)\n",
182 q_properties
->queue_percent
, args
->queue_percentage
);
184 pr_debug("Queue Priority (%d, %d)\n",
185 q_properties
->priority
, args
->queue_priority
);
187 pr_debug("Queue Address (0x%llX, 0x%llX)\n",
188 q_properties
->queue_address
, args
->ring_base_address
);
190 pr_debug("Queue Size (0x%llX, %u)\n",
191 q_properties
->queue_size
, args
->ring_size
);
193 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
194 (uint64_t) q_properties
->read_ptr
,
195 (uint64_t) q_properties
->write_ptr
);
197 pr_debug("Queue Format (%d)\n", q_properties
->format
);
202 static long kfd_ioctl_create_queue(struct file
*filep
, struct kfd_process
*p
,
205 struct kfd_ioctl_create_queue_args args
;
208 unsigned int queue_id
;
209 struct kfd_process_device
*pdd
;
210 struct queue_properties q_properties
;
212 memset(&q_properties
, 0, sizeof(struct queue_properties
));
214 if (copy_from_user(&args
, arg
, sizeof(args
)))
217 pr_debug("kfd: creating queue ioctl\n");
219 err
= set_queue_properties_from_user(&q_properties
, &args
);
223 dev
= kfd_device_by_id(args
.gpu_id
);
227 mutex_lock(&p
->mutex
);
229 pdd
= kfd_bind_process_to_device(dev
, p
);
230 if (IS_ERR(pdd
) < 0) {
232 goto err_bind_process
;
235 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
239 err
= pqm_create_queue(&p
->pqm
, dev
, filep
, &q_properties
, 0,
240 KFD_QUEUE_TYPE_COMPUTE
, &queue_id
);
242 goto err_create_queue
;
244 args
.queue_id
= queue_id
;
246 /* Return gpu_id as doorbell offset for mmap usage */
247 args
.doorbell_offset
= args
.gpu_id
<< PAGE_SHIFT
;
249 if (copy_to_user(arg
, &args
, sizeof(args
))) {
251 goto err_copy_args_out
;
254 mutex_unlock(&p
->mutex
);
256 pr_debug("kfd: queue id %d was created successfully\n", args
.queue_id
);
258 pr_debug("ring buffer address == 0x%016llX\n",
259 args
.ring_base_address
);
261 pr_debug("read ptr address == 0x%016llX\n",
262 args
.read_pointer_address
);
264 pr_debug("write ptr address == 0x%016llX\n",
265 args
.write_pointer_address
);
270 pqm_destroy_queue(&p
->pqm
, queue_id
);
273 mutex_unlock(&p
->mutex
);
277 static int kfd_ioctl_destroy_queue(struct file
*filp
, struct kfd_process
*p
,
281 struct kfd_ioctl_destroy_queue_args args
;
283 if (copy_from_user(&args
, arg
, sizeof(args
)))
286 pr_debug("kfd: destroying queue id %d for PASID %d\n",
290 mutex_lock(&p
->mutex
);
292 retval
= pqm_destroy_queue(&p
->pqm
, args
.queue_id
);
294 mutex_unlock(&p
->mutex
);
298 static int kfd_ioctl_update_queue(struct file
*filp
, struct kfd_process
*p
,
302 struct kfd_ioctl_update_queue_args args
;
303 struct queue_properties properties
;
305 if (copy_from_user(&args
, arg
, sizeof(args
)))
308 if (args
.queue_percentage
> KFD_MAX_QUEUE_PERCENTAGE
) {
309 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
313 if (args
.queue_priority
> KFD_MAX_QUEUE_PRIORITY
) {
314 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
318 if ((args
.ring_base_address
) &&
319 (!access_ok(VERIFY_WRITE
, args
.ring_base_address
, sizeof(uint64_t)))) {
320 pr_err("kfd: can't access ring base address\n");
324 if (!is_power_of_2(args
.ring_size
) && (args
.ring_size
!= 0)) {
325 pr_err("kfd: ring size must be a power of 2 or 0\n");
329 properties
.queue_address
= args
.ring_base_address
;
330 properties
.queue_size
= args
.ring_size
;
331 properties
.queue_percent
= args
.queue_percentage
;
332 properties
.priority
= args
.queue_priority
;
334 pr_debug("kfd: updating queue id %d for PASID %d\n",
335 args
.queue_id
, p
->pasid
);
337 mutex_lock(&p
->mutex
);
339 retval
= pqm_update_queue(&p
->pqm
, args
.queue_id
, &properties
);
341 mutex_unlock(&p
->mutex
);
346 static long kfd_ioctl_set_memory_policy(struct file
*filep
,
347 struct kfd_process
*p
, void __user
*arg
)
349 struct kfd_ioctl_set_memory_policy_args args
;
352 struct kfd_process_device
*pdd
;
353 enum cache_policy default_policy
, alternate_policy
;
355 if (copy_from_user(&args
, arg
, sizeof(args
)))
358 if (args
.default_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
359 && args
.default_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
363 if (args
.alternate_policy
!= KFD_IOC_CACHE_POLICY_COHERENT
364 && args
.alternate_policy
!= KFD_IOC_CACHE_POLICY_NONCOHERENT
) {
368 dev
= kfd_device_by_id(args
.gpu_id
);
372 mutex_lock(&p
->mutex
);
374 pdd
= kfd_bind_process_to_device(dev
, p
);
375 if (IS_ERR(pdd
) < 0) {
380 default_policy
= (args
.default_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
381 ? cache_policy_coherent
: cache_policy_noncoherent
;
384 (args
.alternate_policy
== KFD_IOC_CACHE_POLICY_COHERENT
)
385 ? cache_policy_coherent
: cache_policy_noncoherent
;
387 if (!dev
->dqm
->set_cache_memory_policy(dev
->dqm
,
391 (void __user
*)args
.alternate_aperture_base
,
392 args
.alternate_aperture_size
))
396 mutex_unlock(&p
->mutex
);
401 static long kfd_ioctl_get_clock_counters(struct file
*filep
,
402 struct kfd_process
*p
, void __user
*arg
)
404 struct kfd_ioctl_get_clock_counters_args args
;
406 struct timespec time
;
408 if (copy_from_user(&args
, arg
, sizeof(args
)))
411 dev
= kfd_device_by_id(args
.gpu_id
);
415 /* Reading GPU clock counter from KGD */
416 args
.gpu_clock_counter
= kfd2kgd
->get_gpu_clock_counter(dev
->kgd
);
418 /* No access to rdtsc. Using raw monotonic time */
419 getrawmonotonic(&time
);
420 args
.cpu_clock_counter
= (uint64_t)timespec_to_ns(&time
);
422 get_monotonic_boottime(&time
);
423 args
.system_clock_counter
= (uint64_t)timespec_to_ns(&time
);
425 /* Since the counter is in nano-seconds we use 1GHz frequency */
426 args
.system_clock_freq
= 1000000000;
428 if (copy_to_user(arg
, &args
, sizeof(args
)))
435 static int kfd_ioctl_get_process_apertures(struct file
*filp
,
436 struct kfd_process
*p
, void __user
*arg
)
438 struct kfd_ioctl_get_process_apertures_args args
;
439 struct kfd_process_device_apertures
*pAperture
;
440 struct kfd_process_device
*pdd
;
442 dev_dbg(kfd_device
, "get apertures for PASID %d", p
->pasid
);
444 if (copy_from_user(&args
, arg
, sizeof(args
)))
447 args
.num_of_nodes
= 0;
449 mutex_lock(&p
->mutex
);
451 /*if the process-device list isn't empty*/
452 if (kfd_has_process_device_data(p
)) {
453 /* Run over all pdd of the process */
454 pdd
= kfd_get_first_process_device_data(p
);
456 pAperture
= &args
.process_apertures
[args
.num_of_nodes
];
457 pAperture
->gpu_id
= pdd
->dev
->id
;
458 pAperture
->lds_base
= pdd
->lds_base
;
459 pAperture
->lds_limit
= pdd
->lds_limit
;
460 pAperture
->gpuvm_base
= pdd
->gpuvm_base
;
461 pAperture
->gpuvm_limit
= pdd
->gpuvm_limit
;
462 pAperture
->scratch_base
= pdd
->scratch_base
;
463 pAperture
->scratch_limit
= pdd
->scratch_limit
;
466 "node id %u\n", args
.num_of_nodes
);
468 "gpu id %u\n", pdd
->dev
->id
);
470 "lds_base %llX\n", pdd
->lds_base
);
472 "lds_limit %llX\n", pdd
->lds_limit
);
474 "gpuvm_base %llX\n", pdd
->gpuvm_base
);
476 "gpuvm_limit %llX\n", pdd
->gpuvm_limit
);
478 "scratch_base %llX\n", pdd
->scratch_base
);
480 "scratch_limit %llX\n", pdd
->scratch_limit
);
483 } while ((pdd
= kfd_get_next_process_device_data(p
, pdd
)) != NULL
&&
484 (args
.num_of_nodes
< NUM_OF_SUPPORTED_GPUS
));
487 mutex_unlock(&p
->mutex
);
489 if (copy_to_user(arg
, &args
, sizeof(args
)))
495 static long kfd_ioctl(struct file
*filep
, unsigned int cmd
, unsigned long arg
)
497 struct kfd_process
*process
;
501 "ioctl cmd 0x%x (#%d), arg 0x%lx\n",
502 cmd
, _IOC_NR(cmd
), arg
);
504 process
= kfd_get_process(current
);
506 return PTR_ERR(process
);
509 case KFD_IOC_GET_VERSION
:
510 err
= kfd_ioctl_get_version(filep
, process
, (void __user
*)arg
);
512 case KFD_IOC_CREATE_QUEUE
:
513 err
= kfd_ioctl_create_queue(filep
, process
,
517 case KFD_IOC_DESTROY_QUEUE
:
518 err
= kfd_ioctl_destroy_queue(filep
, process
,
522 case KFD_IOC_SET_MEMORY_POLICY
:
523 err
= kfd_ioctl_set_memory_policy(filep
, process
,
527 case KFD_IOC_GET_CLOCK_COUNTERS
:
528 err
= kfd_ioctl_get_clock_counters(filep
, process
,
532 case KFD_IOC_GET_PROCESS_APERTURES
:
533 err
= kfd_ioctl_get_process_apertures(filep
, process
,
537 case KFD_IOC_UPDATE_QUEUE
:
538 err
= kfd_ioctl_update_queue(filep
, process
,
544 "unknown ioctl cmd 0x%x, arg 0x%lx)\n",
552 "ioctl error %ld for ioctl cmd 0x%x (#%d)\n",
553 err
, cmd
, _IOC_NR(cmd
));
558 static int kfd_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
560 struct kfd_process
*process
;
562 process
= kfd_get_process(current
);
564 return PTR_ERR(process
);
566 return kfd_doorbell_mmap(process
, vma
);