[deliverable/linux.git] / drivers / gpu / drm / amd / amdkfd / kfd_process.c

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include <linux/mutex.h>
#include <linux/log2.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/amd-iommu.h>
#include <linux/notifier.h>
struct mm_struct;

#include "kfd_priv.h"

/*
 * Initial size for the array of queues.
 * The allocated size is doubled each time
 * it is exceeded up to MAX_PROCESS_QUEUES.
 */
#define INITIAL_QUEUE_ARRAY_SIZE 16

/*
 * List of struct kfd_process (field kfd_process).
 * Unique/indexed by mm_struct*
 */
#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
static DEFINE_MUTEX(kfd_processes_mutex);

DEFINE_STATIC_SRCU(kfd_processes_srcu);

static struct workqueue_struct *kfd_process_wq;

struct kfd_process_release_work {
	struct work_struct kfd_work;
	struct kfd_process *p;
};

static struct kfd_process *find_process(const struct task_struct *thread);
static struct kfd_process *create_process(const struct task_struct *thread);

void kfd_process_create_wq(void)
{
	if (!kfd_process_wq)
		kfd_process_wq = create_workqueue("kfd_process_wq");
}

void kfd_process_destroy_wq(void)
{
	if (kfd_process_wq) {
		flush_workqueue(kfd_process_wq);
		destroy_workqueue(kfd_process_wq);
		kfd_process_wq = NULL;
	}
}

struct kfd_process *kfd_create_process(const struct task_struct *thread)
{
	struct kfd_process *process;

	BUG_ON(!kfd_process_wq);

	if (thread->mm == NULL)
		return ERR_PTR(-EINVAL);

	/* Only the pthreads threading model is supported. */
	if (thread->group_leader->mm != thread->mm)
		return ERR_PTR(-EINVAL);

	/* Take mmap_sem because we call __mmu_notifier_register inside */
	down_write(&thread->mm->mmap_sem);

	/*
	 * take kfd processes mutex before starting of process creation
	 * so there won't be a case where two threads of the same process
	 * create two kfd_process structures
	 */
	mutex_lock(&kfd_processes_mutex);

	/* A prior open of /dev/kfd could have already created the process. */
	process = find_process(thread);
	if (process)
		pr_debug("kfd: process already found\n");

	if (!process)
		process = create_process(thread);

	mutex_unlock(&kfd_processes_mutex);

	up_write(&thread->mm->mmap_sem);

	return process;
}

struct kfd_process *kfd_get_process(const struct task_struct *thread)
{
	struct kfd_process *process;

	if (thread->mm == NULL)
		return ERR_PTR(-EINVAL);

	/* Only the pthreads threading model is supported. */
	if (thread->group_leader->mm != thread->mm)
		return ERR_PTR(-EINVAL);

	process = find_process(thread);

	return process;
}

static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
{
	struct kfd_process *process;

	hash_for_each_possible_rcu(kfd_processes_table, process,
					kfd_processes, (uintptr_t)mm)
		if (process->mm == mm)
			return process;

	return NULL;
}

static struct kfd_process *find_process(const struct task_struct *thread)
{
	struct kfd_process *p;
	int idx;

	idx = srcu_read_lock(&kfd_processes_srcu);
	p = find_process_by_mm(thread->mm);
	srcu_read_unlock(&kfd_processes_srcu, idx);

	return p;
}

static void kfd_process_wq_release(struct work_struct *work)
{
	struct kfd_process_release_work *my_work;
	struct kfd_process_device *pdd, *temp;
	struct kfd_process *p;

	my_work = (struct kfd_process_release_work *) work;

	p = my_work->p;

	mutex_lock(&p->mutex);

	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
							per_device_list) {
		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
		list_del(&pdd->per_device_list);

		kfree(pdd);
	}

	kfd_pasid_free(p->pasid);

	mutex_unlock(&p->mutex);

	mutex_destroy(&p->mutex);

	kfree(p->queues);

	kfree(p);

	kfree((void *)work);
}

static void kfd_process_destroy_delayed(struct rcu_head *rcu)
{
	struct kfd_process_release_work *work;
	struct kfd_process *p;

	BUG_ON(!kfd_process_wq);

	p = container_of(rcu, struct kfd_process, rcu);
	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);

	mmdrop(p->mm);

	work = (struct kfd_process_release_work *)
		kmalloc(sizeof(struct kfd_process_release_work), GFP_KERNEL);

	if (work) {
		INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
		work->p = p;
		queue_work(kfd_process_wq, (struct work_struct *) work);
	}
}

static void kfd_process_notifier_release(struct mmu_notifier *mn,
					struct mm_struct *mm)
{
	struct kfd_process *p;

	/*
	 * The kfd_process structure can not be free because the
	 * mmu_notifier srcu is read locked
	 */
	p = container_of(mn, struct kfd_process, mmu_notifier);
	BUG_ON(p->mm != mm);

	mutex_lock(&kfd_processes_mutex);
	hash_del_rcu(&p->kfd_processes);
	mutex_unlock(&kfd_processes_mutex);
	synchronize_srcu(&kfd_processes_srcu);

	/*
	 * Because we drop mm_count inside kfd_process_destroy_delayed
	 * and because the mmu_notifier_unregister function also drop
	 * mm_count we need to take an extra count here.
	 */
	atomic_inc(&p->mm->mm_count);
	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
}

static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
	.release = kfd_process_notifier_release,
};

static struct kfd_process *create_process(const struct task_struct *thread)
{
	struct kfd_process *process;
	int err = -ENOMEM;

	process = kzalloc(sizeof(*process), GFP_KERNEL);

	if (!process)
		goto err_alloc_process;

	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
					sizeof(process->queues[0]), GFP_KERNEL);
	if (!process->queues)
		goto err_alloc_queues;

	process->pasid = kfd_pasid_alloc();
	if (process->pasid == 0)
		goto err_alloc_pasid;

	mutex_init(&process->mutex);

	process->mm = thread->mm;

	/* register notifier */
	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
	if (err)
		goto err_mmu_notifier;

	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
			(uintptr_t)process->mm);

	process->lead_thread = thread->group_leader;

	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;

	INIT_LIST_HEAD(&process->per_device_data);

	return process;

err_mmu_notifier:
	kfd_pasid_free(process->pasid);
err_alloc_pasid:
	kfree(process->queues);
err_alloc_queues:
	kfree(process);
err_alloc_process:
	return ERR_PTR(err);
}

struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
							struct kfd_process *p,
							int create_pdd)
{
	struct kfd_process_device *pdd = NULL;

	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
		if (pdd->dev == dev)
			return pdd;

	if (create_pdd) {
		pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
		if (pdd != NULL) {
			pdd->dev = dev;
			list_add(&pdd->per_device_list, &p->per_device_data);
		}
	}

	return pdd;
}

/*
 * Direct the IOMMU to bind the process (specifically the pasid->mm)
 * to the device.
 * Unbinding occurs when the process dies or the device is removed.
 *
 * Assumes that the process lock is held.
 */
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
							struct kfd_process *p)
{
	struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1);
	int err;

	if (pdd == NULL)
		return ERR_PTR(-ENOMEM);

	if (pdd->bound)
		return pdd;

	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
	if (err < 0)
		return ERR_PTR(err);

	if (err < 0) {
		amd_iommu_unbind_pasid(dev->pdev, p->pasid);
		return ERR_PTR(err);
	}

	pdd->bound = true;

	return pdd;
}

void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
{
	struct kfd_process *p;
	struct kfd_process_device *pdd;
	int idx, i;

	BUG_ON(dev == NULL);

	idx = srcu_read_lock(&kfd_processes_srcu);

	hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
		if (p->pasid == pasid)
			break;

	srcu_read_unlock(&kfd_processes_srcu, idx);

	BUG_ON(p->pasid != pasid);

	mutex_lock(&p->mutex);

	pdd = kfd_get_process_device_data(dev, p, 0);

	/*
	 * Just mark pdd as unbound, because we still need it to call
	 * amd_iommu_unbind_pasid() in when the process exits.
	 * We don't call amd_iommu_unbind_pasid() here
	 * because the IOMMU called us.
	 */
	if (pdd)
		pdd->bound = false;

	mutex_unlock(&p->mutex);
}

struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
{
	return list_first_entry(&p->per_device_data,
				struct kfd_process_device,
				per_device_list);
}

struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
						struct kfd_process_device *pdd)
{
	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
		return NULL;
	return list_next_entry(pdd, per_device_list);
}

bool kfd_has_process_device_data(struct kfd_process *p)
{
	return !(list_empty(&p->per_device_data));
}
Commit	Line	Data
19f6d2a6 OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	#include <linux/mutex.h>
	24	#include <linux/log2.h>
	25	#include <linux/sched.h>
	26	#include <linux/slab.h>
b17f068a	27	#include <linux/amd-iommu.h>
19f6d2a6 OG	28	#include <linux/notifier.h>
	29	struct mm_struct;
	30
	31	#include "kfd_priv.h"
	32
	33	/*
	34	* Initial size for the array of queues.
	35	* The allocated size is doubled each time
	36	* it is exceeded up to MAX_PROCESS_QUEUES.
	37	*/
	38	#define INITIAL_QUEUE_ARRAY_SIZE 16
	39
	40	/*
	41	* List of struct kfd_process (field kfd_process).
	42	* Unique/indexed by mm_struct*
	43	*/
	44	#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
	45	static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
	46	static DEFINE_MUTEX(kfd_processes_mutex);
	47
	48	DEFINE_STATIC_SRCU(kfd_processes_srcu);
	49
	50	static struct workqueue_struct *kfd_process_wq;
	51
	52	struct kfd_process_release_work {
	53	struct work_struct kfd_work;
	54	struct kfd_process *p;
	55	};
	56
	57	static struct kfd_process find_process(const struct task_struct thread);
	58	static struct kfd_process create_process(const struct task_struct thread);
	59
	60	void kfd_process_create_wq(void)
	61	{
	62	if (!kfd_process_wq)
	63	kfd_process_wq = create_workqueue("kfd_process_wq");
	64	}
	65
	66	void kfd_process_destroy_wq(void)
	67	{
	68	if (kfd_process_wq) {
	69	flush_workqueue(kfd_process_wq);
	70	destroy_workqueue(kfd_process_wq);
	71	kfd_process_wq = NULL;
	72	}
	73	}
	74
	75	struct kfd_process kfd_create_process(const struct task_struct thread)
	76	{
	77	struct kfd_process *process;
	78
	79	BUG_ON(!kfd_process_wq);
	80
	81	if (thread->mm == NULL)
	82	return ERR_PTR(-EINVAL);
	83
	84	/* Only the pthreads threading model is supported. */
	85	if (thread->group_leader->mm != thread->mm)
	86	return ERR_PTR(-EINVAL);
	87
	88	/* Take mmap_sem because we call __mmu_notifier_register inside */
	89	down_write(&thread->mm->mmap_sem);
	90
	91	/*
92	* take kfd processes mutex before starting of process creation
93	* so there won't be a case where two threads of the same process
94	* create two kfd_process structures
95	*/
96	mutex_lock(&kfd_processes_mutex);
97
98	/* A prior open of /dev/kfd could have already created the process. */
99	process = find_process(thread);
100	if (process)
101	pr_debug("kfd: process already found\n");
102
103	if (!process)
104	process = create_process(thread);
105
106	mutex_unlock(&kfd_processes_mutex);
107
108	up_write(&thread->mm->mmap_sem);
109
110	return process;
111	}
112
113	struct kfd_process kfd_get_process(const struct task_struct thread)
114	{
115	struct kfd_process *process;
116
117	if (thread->mm == NULL)
118	return ERR_PTR(-EINVAL);
119
120	/* Only the pthreads threading model is supported. */
121	if (thread->group_leader->mm != thread->mm)
122	return ERR_PTR(-EINVAL);
123
124	process = find_process(thread);
125
126	return process;
127	}
128
129	static struct kfd_process find_process_by_mm(const struct mm_struct mm)
130	{
131	struct kfd_process *process;
132
133	hash_for_each_possible_rcu(kfd_processes_table, process,
134	kfd_processes, (uintptr_t)mm)
135	if (process->mm == mm)
136	return process;
137
138	return NULL;
139	}
140
141	static struct kfd_process find_process(const struct task_struct thread)
142	{
143	struct kfd_process *p;
144	int idx;
145
146	idx = srcu_read_lock(&kfd_processes_srcu);
147	p = find_process_by_mm(thread->mm);
148	srcu_read_unlock(&kfd_processes_srcu, idx);
149
150	return p;
151	}
152
153	static void kfd_process_wq_release(struct work_struct *work)
154	{
155	struct kfd_process_release_work *my_work;
156	struct kfd_process_device pdd, temp;
157	struct kfd_process *p;
158
159	my_work = (struct kfd_process_release_work *) work;
160
161	p = my_work->p;
162
163	mutex_lock(&p->mutex);
164
165	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
166	per_device_list) {
b17f068a	167	amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
19f6d2a6 OG	168	list_del(&pdd->per_device_list);
	169
	170	kfree(pdd);
	171	}
	172
	173	kfd_pasid_free(p->pasid);
	174
	175	mutex_unlock(&p->mutex);
	176
	177	mutex_destroy(&p->mutex);
	178
	179	kfree(p->queues);
	180
	181	kfree(p);
	182
	183	kfree((void *)work);
	184	}
	185
	186	static void kfd_process_destroy_delayed(struct rcu_head *rcu)
	187	{
	188	struct kfd_process_release_work *work;
	189	struct kfd_process *p;
	190
	191	BUG_ON(!kfd_process_wq);
	192
	193	p = container_of(rcu, struct kfd_process, rcu);
	194	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
	195
	196	mmdrop(p->mm);
	197
	198	work = (struct kfd_process_release_work *)
	199	kmalloc(sizeof(struct kfd_process_release_work), GFP_KERNEL);
	200
	201	if (work) {
	202	INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
	203	work->p = p;
	204	queue_work(kfd_process_wq, (struct work_struct *) work);
	205	}
	206	}
	207
	208	static void kfd_process_notifier_release(struct mmu_notifier *mn,
	209	struct mm_struct *mm)
	210	{
	211	struct kfd_process *p;
	212
	213	/*
	214	* The kfd_process structure can not be free because the
	215	* mmu_notifier srcu is read locked
	216	*/
	217	p = container_of(mn, struct kfd_process, mmu_notifier);
	218	BUG_ON(p->mm != mm);
	219
	220	mutex_lock(&kfd_processes_mutex);
	221	hash_del_rcu(&p->kfd_processes);
	222	mutex_unlock(&kfd_processes_mutex);
	223	synchronize_srcu(&kfd_processes_srcu);
	224
	225	/*
	226	* Because we drop mm_count inside kfd_process_destroy_delayed
	227	* and because the mmu_notifier_unregister function also drop
	228	* mm_count we need to take an extra count here.
	229	*/
	230	atomic_inc(&p->mm->mm_count);
	231	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
232	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
233	}
234
235	static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
236	.release = kfd_process_notifier_release,
237	};
238
239	static struct kfd_process create_process(const struct task_struct thread)
240	{
241	struct kfd_process *process;
242	int err = -ENOMEM;
243
244	process = kzalloc(sizeof(*process), GFP_KERNEL);
245
246	if (!process)
247	goto err_alloc_process;
248
249	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
250	sizeof(process->queues[0]), GFP_KERNEL);
251	if (!process->queues)
252	goto err_alloc_queues;
253
254	process->pasid = kfd_pasid_alloc();
255	if (process->pasid == 0)
256	goto err_alloc_pasid;
257
258	mutex_init(&process->mutex);
259
260	process->mm = thread->mm;
261
262	/* register notifier */
263	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
264	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
265	if (err)
266	goto err_mmu_notifier;
267
268	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
269	(uintptr_t)process->mm);
270
271	process->lead_thread = thread->group_leader;
272
273	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
274
275	INIT_LIST_HEAD(&process->per_device_data);
276
277	return process;
278
279	err_mmu_notifier:
280	kfd_pasid_free(process->pasid);
281	err_alloc_pasid:
282	kfree(process->queues);
283	err_alloc_queues:
284	kfree(process);
285	err_alloc_process:
286	return ERR_PTR(err);
287	}
288
289	struct kfd_process_device kfd_get_process_device_data(struct kfd_dev dev,
290	struct kfd_process *p,
291	int create_pdd)
292	{
293	struct kfd_process_device *pdd = NULL;
294
295	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
296	if (pdd->dev == dev)
297	return pdd;
298
299	if (create_pdd) {
300	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
301	if (pdd != NULL) {
302	pdd->dev = dev;
303	list_add(&pdd->per_device_list, &p->per_device_data);
304	}
305	}
306
307	return pdd;
308	}
309
310	/*
311	* Direct the IOMMU to bind the process (specifically the pasid->mm)
312	* to the device.
313	* Unbinding occurs when the process dies or the device is removed.
314	*
315	* Assumes that the process lock is held.
316	*/
317	struct kfd_process_device kfd_bind_process_to_device(struct kfd_dev dev,
318	struct kfd_process *p)
319	{
320	struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1);
b17f068a	321	int err;
19f6d2a6 OG	322
	323	if (pdd == NULL)
	324	return ERR_PTR(-ENOMEM);
	325
	326	if (pdd->bound)
	327	return pdd;
	328
b17f068a OG	329	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
	330	if (err < 0)
	331	return ERR_PTR(err);
	332
	333	if (err < 0) {
	334	amd_iommu_unbind_pasid(dev->pdev, p->pasid);
	335	return ERR_PTR(err);
	336	}
	337
19f6d2a6 OG	338	pdd->bound = true;
	339
	340	return pdd;
	341	}
	342
	343	void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
	344	{
	345	struct kfd_process *p;
	346	struct kfd_process_device *pdd;
	347	int idx, i;
	348
	349	BUG_ON(dev == NULL);
	350
	351	idx = srcu_read_lock(&kfd_processes_srcu);
	352
	353	hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
	354	if (p->pasid == pasid)
	355	break;
	356
	357	srcu_read_unlock(&kfd_processes_srcu, idx);
	358
	359	BUG_ON(p->pasid != pasid);
	360
	361	mutex_lock(&p->mutex);
	362
	363	pdd = kfd_get_process_device_data(dev, p, 0);
	364
	365	/*
	366	* Just mark pdd as unbound, because we still need it to call
	367	* amd_iommu_unbind_pasid() in when the process exits.
	368	* We don't call amd_iommu_unbind_pasid() here
	369	* because the IOMMU called us.
	370	*/
	371	if (pdd)
	372	pdd->bound = false;
	373
	374	mutex_unlock(&p->mutex);
	375	}
	376
	377	struct kfd_process_device kfd_get_first_process_device_data(struct kfd_process p)
	378	{
	379	return list_first_entry(&p->per_device_data,
	380	struct kfd_process_device,
	381	per_device_list);
	382	}
	383
	384	struct kfd_process_device kfd_get_next_process_device_data(struct kfd_process p,
	385	struct kfd_process_device *pdd)
	386	{
	387	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
	388	return NULL;
	389	return list_next_entry(pdd, per_device_list);
	390	}
	391
	392	bool kfd_has_process_device_data(struct kfd_process *p)
	393	{
	394	return !(list_empty(&p->per_device_data));
	395	}