2 * Copyright 2015 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/kthread.h>
25 #include <linux/wait.h>
26 #include <linux/sched.h>
28 #include "gpu_scheduler.h"
30 #define CREATE_TRACE_POINTS
31 #include "gpu_sched_trace.h"
33 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
);
34 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
);
36 struct kmem_cache
*sched_fence_slab
;
37 atomic_t sched_fence_slab_ref
= ATOMIC_INIT(0);
39 /* Initialize a given run queue struct */
40 static void amd_sched_rq_init(struct amd_sched_rq
*rq
)
42 spin_lock_init(&rq
->lock
);
43 INIT_LIST_HEAD(&rq
->entities
);
44 rq
->current_entity
= NULL
;
47 static void amd_sched_rq_add_entity(struct amd_sched_rq
*rq
,
48 struct amd_sched_entity
*entity
)
50 if (!list_empty(&entity
->list
))
53 list_add_tail(&entity
->list
, &rq
->entities
);
54 spin_unlock(&rq
->lock
);
57 static void amd_sched_rq_remove_entity(struct amd_sched_rq
*rq
,
58 struct amd_sched_entity
*entity
)
60 if (list_empty(&entity
->list
))
63 list_del_init(&entity
->list
);
64 if (rq
->current_entity
== entity
)
65 rq
->current_entity
= NULL
;
66 spin_unlock(&rq
->lock
);
70 * Select an entity which could provide a job to run
72 * @rq The run queue to check.
74 * Try to find a ready entity, returns NULL if none found.
76 static struct amd_sched_entity
*
77 amd_sched_rq_select_entity(struct amd_sched_rq
*rq
)
79 struct amd_sched_entity
*entity
;
83 entity
= rq
->current_entity
;
85 list_for_each_entry_continue(entity
, &rq
->entities
, list
) {
86 if (amd_sched_entity_is_ready(entity
)) {
87 rq
->current_entity
= entity
;
88 spin_unlock(&rq
->lock
);
94 list_for_each_entry(entity
, &rq
->entities
, list
) {
96 if (amd_sched_entity_is_ready(entity
)) {
97 rq
->current_entity
= entity
;
98 spin_unlock(&rq
->lock
);
102 if (entity
== rq
->current_entity
)
106 spin_unlock(&rq
->lock
);
112 * Init a context entity used by scheduler when submit to HW ring.
114 * @sched The pointer to the scheduler
115 * @entity The pointer to a valid amd_sched_entity
116 * @rq The run queue this entity belongs
117 * @kernel If this is an entity for the kernel
118 * @jobs The max number of jobs in the job queue
120 * return 0 if succeed. negative error code on failure
122 int amd_sched_entity_init(struct amd_gpu_scheduler
*sched
,
123 struct amd_sched_entity
*entity
,
124 struct amd_sched_rq
*rq
,
129 if (!(sched
&& entity
&& rq
))
132 memset(entity
, 0, sizeof(struct amd_sched_entity
));
133 INIT_LIST_HEAD(&entity
->list
);
135 entity
->sched
= sched
;
137 spin_lock_init(&entity
->queue_lock
);
138 r
= kfifo_alloc(&entity
->job_queue
, jobs
* sizeof(void *), GFP_KERNEL
);
142 atomic_set(&entity
->fence_seq
, 0);
143 entity
->fence_context
= fence_context_alloc(1);
149 * Query if entity is initialized
151 * @sched Pointer to scheduler instance
152 * @entity The pointer to a valid scheduler entity
154 * return true if entity is initialized, false otherwise
156 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler
*sched
,
157 struct amd_sched_entity
*entity
)
159 return entity
->sched
== sched
&&
164 * Check if entity is idle
166 * @entity The pointer to a valid scheduler entity
168 * Return true if entity don't has any unscheduled jobs.
170 static bool amd_sched_entity_is_idle(struct amd_sched_entity
*entity
)
173 if (kfifo_is_empty(&entity
->job_queue
))
180 * Check if entity is ready
182 * @entity The pointer to a valid scheduler entity
184 * Return true if entity could provide a job.
186 static bool amd_sched_entity_is_ready(struct amd_sched_entity
*entity
)
188 if (kfifo_is_empty(&entity
->job_queue
))
191 if (ACCESS_ONCE(entity
->dependency
))
198 * Destroy a context entity
200 * @sched Pointer to scheduler instance
201 * @entity The pointer to a valid scheduler entity
203 * Cleanup and free the allocated resources.
205 void amd_sched_entity_fini(struct amd_gpu_scheduler
*sched
,
206 struct amd_sched_entity
*entity
)
208 struct amd_sched_rq
*rq
= entity
->rq
;
210 if (!amd_sched_entity_is_initialized(sched
, entity
))
214 * The client will not queue more IBs during this fini, consume existing
217 wait_event(sched
->job_scheduled
, amd_sched_entity_is_idle(entity
));
219 amd_sched_rq_remove_entity(rq
, entity
);
220 kfifo_free(&entity
->job_queue
);
223 static void amd_sched_entity_wakeup(struct fence
*f
, struct fence_cb
*cb
)
225 struct amd_sched_entity
*entity
=
226 container_of(cb
, struct amd_sched_entity
, cb
);
227 entity
->dependency
= NULL
;
229 amd_sched_wakeup(entity
->sched
);
232 static void amd_sched_entity_clear_dep(struct fence
*f
, struct fence_cb
*cb
)
234 struct amd_sched_entity
*entity
=
235 container_of(cb
, struct amd_sched_entity
, cb
);
236 entity
->dependency
= NULL
;
240 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity
*entity
)
242 struct amd_gpu_scheduler
*sched
= entity
->sched
;
243 struct fence
* fence
= entity
->dependency
;
244 struct amd_sched_fence
*s_fence
;
246 if (fence
->context
== entity
->fence_context
) {
247 /* We can ignore fences from ourself */
248 fence_put(entity
->dependency
);
252 s_fence
= to_amd_sched_fence(fence
);
253 if (s_fence
&& s_fence
->sched
== sched
) {
254 /* Fence is from the same scheduler */
255 if (test_bit(AMD_SCHED_FENCE_SCHEDULED_BIT
, &fence
->flags
)) {
256 /* Ignore it when it is already scheduled */
257 fence_put(entity
->dependency
);
261 /* Wait for fence to be scheduled */
262 entity
->cb
.func
= amd_sched_entity_clear_dep
;
263 list_add_tail(&entity
->cb
.node
, &s_fence
->scheduled_cb
);
267 if (!fence_add_callback(entity
->dependency
, &entity
->cb
,
268 amd_sched_entity_wakeup
))
271 fence_put(entity
->dependency
);
275 static struct amd_sched_job
*
276 amd_sched_entity_pop_job(struct amd_sched_entity
*entity
)
278 struct amd_gpu_scheduler
*sched
= entity
->sched
;
279 struct amd_sched_job
*sched_job
;
281 if (!kfifo_out_peek(&entity
->job_queue
, &sched_job
, sizeof(sched_job
)))
284 while ((entity
->dependency
= sched
->ops
->dependency(sched_job
)))
285 if (amd_sched_entity_add_dependency_cb(entity
))
292 * Helper to submit a job to the job queue
294 * @sched_job The pointer to job required to submit
296 * Returns true if we could submit the job.
298 static bool amd_sched_entity_in(struct amd_sched_job
*sched_job
)
300 struct amd_gpu_scheduler
*sched
= sched_job
->sched
;
301 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
302 bool added
, first
= false;
304 spin_lock(&entity
->queue_lock
);
305 added
= kfifo_in(&entity
->job_queue
, &sched_job
,
306 sizeof(sched_job
)) == sizeof(sched_job
);
308 if (added
&& kfifo_len(&entity
->job_queue
) == sizeof(sched_job
))
311 spin_unlock(&entity
->queue_lock
);
313 /* first job wakes up scheduler */
315 /* Add the entity to the run queue */
316 amd_sched_rq_add_entity(entity
->rq
, entity
);
317 amd_sched_wakeup(sched
);
322 static void amd_sched_free_job(struct fence
*f
, struct fence_cb
*cb
) {
323 struct amd_sched_job
*job
= container_of(cb
, struct amd_sched_job
, cb_free_job
);
324 schedule_work(&job
->work_free_job
);
327 /* job_finish is called after hw fence signaled, and
328 * the job had already been deleted from ring_mirror_list
330 void amd_sched_job_finish(struct amd_sched_job
*s_job
)
332 struct amd_sched_job
*next
;
333 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
335 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
) {
336 if (cancel_delayed_work(&s_job
->work_tdr
))
337 amd_sched_job_put(s_job
);
339 /* queue TDR for next job */
340 next
= list_first_entry_or_null(&sched
->ring_mirror_list
,
341 struct amd_sched_job
, node
);
344 INIT_DELAYED_WORK(&next
->work_tdr
, s_job
->timeout_callback
);
345 amd_sched_job_get(next
);
346 schedule_delayed_work(&next
->work_tdr
, sched
->timeout
);
351 void amd_sched_job_begin(struct amd_sched_job
*s_job
)
353 struct amd_gpu_scheduler
*sched
= s_job
->sched
;
355 if (sched
->timeout
!= MAX_SCHEDULE_TIMEOUT
&&
356 list_first_entry_or_null(&sched
->ring_mirror_list
, struct amd_sched_job
, node
) == s_job
)
358 INIT_DELAYED_WORK(&s_job
->work_tdr
, s_job
->timeout_callback
);
359 amd_sched_job_get(s_job
);
360 schedule_delayed_work(&s_job
->work_tdr
, sched
->timeout
);
365 * Submit a job to the job queue
367 * @sched_job The pointer to job required to submit
369 * Returns 0 for success, negative error code otherwise.
371 void amd_sched_entity_push_job(struct amd_sched_job
*sched_job
)
373 struct amd_sched_entity
*entity
= sched_job
->s_entity
;
375 sched_job
->use_sched
= 1;
376 fence_add_callback(&sched_job
->s_fence
->base
,
377 &sched_job
->cb_free_job
, amd_sched_free_job
);
378 trace_amd_sched_job(sched_job
);
379 wait_event(entity
->sched
->job_scheduled
,
380 amd_sched_entity_in(sched_job
));
383 /* init a sched_job with basic field */
384 int amd_sched_job_init(struct amd_sched_job
*job
,
385 struct amd_gpu_scheduler
*sched
,
386 struct amd_sched_entity
*entity
,
387 void (*timeout_cb
)(struct work_struct
*work
),
388 void (*free_cb
)(struct kref
*refcount
),
389 void *owner
, struct fence
**fence
)
391 INIT_LIST_HEAD(&job
->node
);
392 kref_init(&job
->refcount
);
394 job
->s_entity
= entity
;
395 job
->s_fence
= amd_sched_fence_create(entity
, owner
);
399 job
->s_fence
->s_job
= job
;
400 job
->timeout_callback
= timeout_cb
;
401 job
->free_callback
= free_cb
;
404 *fence
= &job
->s_fence
->base
;
409 * Return ture if we can push more jobs to the hw.
411 static bool amd_sched_ready(struct amd_gpu_scheduler
*sched
)
413 return atomic_read(&sched
->hw_rq_count
) <
414 sched
->hw_submission_limit
;
418 * Wake up the scheduler when it is ready
420 static void amd_sched_wakeup(struct amd_gpu_scheduler
*sched
)
422 if (amd_sched_ready(sched
))
423 wake_up_interruptible(&sched
->wake_up_worker
);
427 * Select next entity to process
429 static struct amd_sched_entity
*
430 amd_sched_select_entity(struct amd_gpu_scheduler
*sched
)
432 struct amd_sched_entity
*entity
;
435 if (!amd_sched_ready(sched
))
438 /* Kernel run queue has higher priority than normal run queue*/
439 for (i
= 0; i
< AMD_SCHED_MAX_PRIORITY
; i
++) {
440 entity
= amd_sched_rq_select_entity(&sched
->sched_rq
[i
]);
448 static void amd_sched_process_job(struct fence
*f
, struct fence_cb
*cb
)
450 struct amd_sched_fence
*s_fence
=
451 container_of(cb
, struct amd_sched_fence
, cb
);
452 struct amd_gpu_scheduler
*sched
= s_fence
->sched
;
455 atomic_dec(&sched
->hw_rq_count
);
457 /* remove job from ring_mirror_list */
458 spin_lock_irqsave(&sched
->job_list_lock
, flags
);
459 list_del_init(&s_fence
->s_job
->node
);
460 sched
->ops
->finish_job(s_fence
->s_job
);
461 spin_unlock_irqrestore(&sched
->job_list_lock
, flags
);
463 amd_sched_fence_signal(s_fence
);
465 trace_amd_sched_process_job(s_fence
);
466 fence_put(&s_fence
->base
);
467 wake_up_interruptible(&sched
->wake_up_worker
);
470 static int amd_sched_main(void *param
)
472 struct sched_param sparam
= {.sched_priority
= 1};
473 struct amd_gpu_scheduler
*sched
= (struct amd_gpu_scheduler
*)param
;
476 sched_setscheduler(current
, SCHED_FIFO
, &sparam
);
478 while (!kthread_should_stop()) {
479 struct amd_sched_entity
*entity
;
480 struct amd_sched_fence
*s_fence
;
481 struct amd_sched_job
*sched_job
;
484 wait_event_interruptible(sched
->wake_up_worker
,
485 (entity
= amd_sched_select_entity(sched
)) ||
486 kthread_should_stop());
491 sched_job
= amd_sched_entity_pop_job(entity
);
495 s_fence
= sched_job
->s_fence
;
497 atomic_inc(&sched
->hw_rq_count
);
498 amd_sched_job_pre_schedule(sched
, sched_job
);
499 fence
= sched
->ops
->run_job(sched_job
);
500 amd_sched_fence_scheduled(s_fence
);
502 r
= fence_add_callback(fence
, &s_fence
->cb
,
503 amd_sched_process_job
);
505 amd_sched_process_job(fence
, &s_fence
->cb
);
507 DRM_ERROR("fence add callback failed (%d)\n", r
);
510 DRM_ERROR("Failed to run job!\n");
511 amd_sched_process_job(NULL
, &s_fence
->cb
);
514 count
= kfifo_out(&entity
->job_queue
, &sched_job
,
516 WARN_ON(count
!= sizeof(sched_job
));
517 wake_up(&sched
->job_scheduled
);
523 * Init a gpu scheduler instance
525 * @sched The pointer to the scheduler
526 * @ops The backend operations for this scheduler.
527 * @hw_submissions Number of hw submissions to do.
528 * @name Name used for debugging
530 * Return 0 on success, otherwise error code.
532 int amd_sched_init(struct amd_gpu_scheduler
*sched
,
533 const struct amd_sched_backend_ops
*ops
,
534 unsigned hw_submission
, long timeout
, const char *name
)
538 sched
->hw_submission_limit
= hw_submission
;
540 sched
->timeout
= timeout
;
541 for (i
= 0; i
< AMD_SCHED_MAX_PRIORITY
; i
++)
542 amd_sched_rq_init(&sched
->sched_rq
[i
]);
544 init_waitqueue_head(&sched
->wake_up_worker
);
545 init_waitqueue_head(&sched
->job_scheduled
);
546 INIT_LIST_HEAD(&sched
->ring_mirror_list
);
547 spin_lock_init(&sched
->job_list_lock
);
548 atomic_set(&sched
->hw_rq_count
, 0);
549 if (atomic_inc_return(&sched_fence_slab_ref
) == 1) {
550 sched_fence_slab
= kmem_cache_create(
551 "amd_sched_fence", sizeof(struct amd_sched_fence
), 0,
552 SLAB_HWCACHE_ALIGN
, NULL
);
553 if (!sched_fence_slab
)
557 /* Each scheduler will run on a seperate kernel thread */
558 sched
->thread
= kthread_run(amd_sched_main
, sched
, sched
->name
);
559 if (IS_ERR(sched
->thread
)) {
560 DRM_ERROR("Failed to create scheduler for %s.\n", name
);
561 return PTR_ERR(sched
->thread
);
568 * Destroy a gpu scheduler
570 * @sched The pointer to the scheduler
572 void amd_sched_fini(struct amd_gpu_scheduler
*sched
)
575 kthread_stop(sched
->thread
);
576 if (atomic_dec_and_test(&sched_fence_slab_ref
))
577 kmem_cache_destroy(sched_fence_slab
);