1 /* sched.c - SPU scheduler.
3 * Copyright (C) IBM 2005
4 * Author: Mark Nutter <mnutter@us.ibm.com>
6 * 2006-03-31 NUMA domains added.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 #include <linux/module.h>
26 #include <linux/errno.h>
27 #include <linux/sched.h>
28 #include <linux/kernel.h>
30 #include <linux/completion.h>
31 #include <linux/vmalloc.h>
32 #include <linux/smp.h>
33 #include <linux/stddef.h>
34 #include <linux/unistd.h>
35 #include <linux/numa.h>
36 #include <linux/mutex.h>
37 #include <linux/notifier.h>
38 #include <linux/kthread.h>
39 #include <linux/pid_namespace.h>
40 #include <linux/proc_fs.h>
41 #include <linux/seq_file.h>
44 #include <asm/mmu_context.h>
46 #include <asm/spu_csa.h>
47 #include <asm/spu_priv1.h>
50 struct spu_prio_array
{
51 DECLARE_BITMAP(bitmap
, MAX_PRIO
);
52 struct list_head runq
[MAX_PRIO
];
57 static unsigned long spu_avenrun
[3];
58 static struct spu_prio_array
*spu_prio
;
59 static struct task_struct
*spusched_task
;
60 static struct timer_list spusched_timer
;
63 * Priority of a normal, non-rt, non-niced'd process (aka nice level 0).
65 #define NORMAL_PRIO 120
68 * Frequency of the spu scheduler tick. By default we do one SPU scheduler
69 * tick for every 10 CPU scheduler ticks.
71 #define SPUSCHED_TICK (10)
74 * These are the 'tuning knobs' of the scheduler:
76 * Minimum timeslice is 5 msecs (or 1 spu scheduler tick, whichever is
77 * larger), default timeslice is 100 msecs, maximum timeslice is 800 msecs.
79 #define MIN_SPU_TIMESLICE max(5 * HZ / (1000 * SPUSCHED_TICK), 1)
80 #define DEF_SPU_TIMESLICE (100 * HZ / (1000 * SPUSCHED_TICK))
82 #define MAX_USER_PRIO (MAX_PRIO - MAX_RT_PRIO)
83 #define SCALE_PRIO(x, prio) \
84 max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_SPU_TIMESLICE)
87 * scale user-nice values [ -20 ... 0 ... 19 ] to time slice values:
88 * [800ms ... 100ms ... 5ms]
90 * The higher a thread's priority, the bigger timeslices
91 * it gets during one round of execution. But even the lowest
92 * priority thread gets MIN_TIMESLICE worth of execution time.
94 void spu_set_timeslice(struct spu_context
*ctx
)
96 if (ctx
->prio
< NORMAL_PRIO
)
97 ctx
->time_slice
= SCALE_PRIO(DEF_SPU_TIMESLICE
* 4, ctx
->prio
);
99 ctx
->time_slice
= SCALE_PRIO(DEF_SPU_TIMESLICE
, ctx
->prio
);
103 * Update scheduling information from the owning thread.
105 void __spu_update_sched_info(struct spu_context
*ctx
)
108 * 32-Bit assignment are atomic on powerpc, and we don't care about
109 * memory ordering here because retriving the controlling thread is
110 * per defintion racy.
112 ctx
->tid
= current
->pid
;
115 * We do our own priority calculations, so we normally want
116 * ->static_prio to start with. Unfortunately thies field
117 * contains junk for threads with a realtime scheduling
118 * policy so we have to look at ->prio in this case.
120 if (rt_prio(current
->prio
))
121 ctx
->prio
= current
->prio
;
123 ctx
->prio
= current
->static_prio
;
124 ctx
->policy
= current
->policy
;
127 * A lot of places that don't hold list_mutex poke into
128 * cpus_allowed, including grab_runnable_context which
129 * already holds the runq_lock. So abuse runq_lock
130 * to protect this field aswell.
132 spin_lock(&spu_prio
->runq_lock
);
133 ctx
->cpus_allowed
= current
->cpus_allowed
;
134 spin_unlock(&spu_prio
->runq_lock
);
137 void spu_update_sched_info(struct spu_context
*ctx
)
139 int node
= ctx
->spu
->node
;
141 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
142 __spu_update_sched_info(ctx
);
143 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
146 static int __node_allowed(struct spu_context
*ctx
, int node
)
148 if (nr_cpus_node(node
)) {
149 cpumask_t mask
= node_to_cpumask(node
);
151 if (cpus_intersects(mask
, ctx
->cpus_allowed
))
158 static int node_allowed(struct spu_context
*ctx
, int node
)
162 spin_lock(&spu_prio
->runq_lock
);
163 rval
= __node_allowed(ctx
, node
);
164 spin_unlock(&spu_prio
->runq_lock
);
169 static BLOCKING_NOTIFIER_HEAD(spu_switch_notifier
);
171 void spu_switch_notify(struct spu
*spu
, struct spu_context
*ctx
)
173 blocking_notifier_call_chain(&spu_switch_notifier
,
174 ctx
? ctx
->object_id
: 0, spu
);
177 static void notify_spus_active(void)
182 * Wake up the active spu_contexts.
184 * When the awakened processes see their "notify_active" flag is set,
185 * they will call spu_switch_notify();
187 for_each_online_node(node
) {
190 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
191 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
192 if (spu
->alloc_state
!= SPU_FREE
) {
193 struct spu_context
*ctx
= spu
->ctx
;
194 set_bit(SPU_SCHED_NOTIFY_ACTIVE
,
197 wake_up_all(&ctx
->stop_wq
);
200 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
204 int spu_switch_event_register(struct notifier_block
* n
)
207 ret
= blocking_notifier_chain_register(&spu_switch_notifier
, n
);
209 notify_spus_active();
212 EXPORT_SYMBOL_GPL(spu_switch_event_register
);
214 int spu_switch_event_unregister(struct notifier_block
* n
)
216 return blocking_notifier_chain_unregister(&spu_switch_notifier
, n
);
218 EXPORT_SYMBOL_GPL(spu_switch_event_unregister
);
221 * spu_bind_context - bind spu context to physical spu
222 * @spu: physical spu to bind to
223 * @ctx: context to bind
225 static void spu_bind_context(struct spu
*spu
, struct spu_context
*ctx
)
227 pr_debug("%s: pid=%d SPU=%d NODE=%d\n", __FUNCTION__
, current
->pid
,
228 spu
->number
, spu
->node
);
229 spuctx_switch_state(ctx
, SPU_UTIL_SYSTEM
);
231 if (ctx
->flags
& SPU_CREATE_NOSCHED
)
232 atomic_inc(&cbe_spu_info
[spu
->node
].reserved_spus
);
233 if (!list_empty(&ctx
->aff_list
))
234 atomic_inc(&ctx
->gang
->aff_sched_count
);
236 ctx
->stats
.slb_flt_base
= spu
->stats
.slb_flt
;
237 ctx
->stats
.class2_intr_base
= spu
->stats
.class2_intr
;
242 ctx
->ops
= &spu_hw_ops
;
243 spu
->pid
= current
->pid
;
244 spu
->tgid
= current
->tgid
;
245 spu_associate_mm(spu
, ctx
->owner
);
246 spu
->ibox_callback
= spufs_ibox_callback
;
247 spu
->wbox_callback
= spufs_wbox_callback
;
248 spu
->stop_callback
= spufs_stop_callback
;
249 spu
->mfc_callback
= spufs_mfc_callback
;
250 spu
->dma_callback
= spufs_dma_callback
;
252 spu_unmap_mappings(ctx
);
253 spu_restore(&ctx
->csa
, spu
);
254 spu
->timestamp
= jiffies
;
255 spu_cpu_affinity_set(spu
, raw_smp_processor_id());
256 spu_switch_notify(spu
, ctx
);
257 ctx
->state
= SPU_STATE_RUNNABLE
;
259 spuctx_switch_state(ctx
, SPU_UTIL_IDLE_LOADED
);
263 * Must be used with the list_mutex held.
265 static inline int sched_spu(struct spu
*spu
)
267 BUG_ON(!mutex_is_locked(&cbe_spu_info
[spu
->node
].list_mutex
));
269 return (!spu
->ctx
|| !(spu
->ctx
->flags
& SPU_CREATE_NOSCHED
));
272 static void aff_merge_remaining_ctxs(struct spu_gang
*gang
)
274 struct spu_context
*ctx
;
276 list_for_each_entry(ctx
, &gang
->aff_list_head
, aff_list
) {
277 if (list_empty(&ctx
->aff_list
))
278 list_add(&ctx
->aff_list
, &gang
->aff_list_head
);
280 gang
->aff_flags
|= AFF_MERGED
;
283 static void aff_set_offsets(struct spu_gang
*gang
)
285 struct spu_context
*ctx
;
289 list_for_each_entry_reverse(ctx
, &gang
->aff_ref_ctx
->aff_list
,
291 if (&ctx
->aff_list
== &gang
->aff_list_head
)
293 ctx
->aff_offset
= offset
--;
297 list_for_each_entry(ctx
, gang
->aff_ref_ctx
->aff_list
.prev
, aff_list
) {
298 if (&ctx
->aff_list
== &gang
->aff_list_head
)
300 ctx
->aff_offset
= offset
++;
303 gang
->aff_flags
|= AFF_OFFSETS_SET
;
306 static struct spu
*aff_ref_location(struct spu_context
*ctx
, int mem_aff
,
307 int group_size
, int lowest_offset
)
313 * TODO: A better algorithm could be used to find a good spu to be
314 * used as reference location for the ctxs chain.
316 node
= cpu_to_node(raw_smp_processor_id());
317 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
318 node
= (node
< MAX_NUMNODES
) ? node
: 0;
319 if (!node_allowed(ctx
, node
))
321 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
322 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
323 if ((!mem_aff
|| spu
->has_mem_affinity
) &&
325 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
329 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
334 static void aff_set_ref_point_location(struct spu_gang
*gang
)
336 int mem_aff
, gs
, lowest_offset
;
337 struct spu_context
*ctx
;
340 mem_aff
= gang
->aff_ref_ctx
->flags
& SPU_CREATE_AFFINITY_MEM
;
344 list_for_each_entry(tmp
, &gang
->aff_list_head
, aff_list
)
347 list_for_each_entry_reverse(ctx
, &gang
->aff_ref_ctx
->aff_list
,
349 if (&ctx
->aff_list
== &gang
->aff_list_head
)
351 lowest_offset
= ctx
->aff_offset
;
354 gang
->aff_ref_spu
= aff_ref_location(ctx
, mem_aff
, gs
, lowest_offset
);
357 static struct spu
*ctx_location(struct spu
*ref
, int offset
, int node
)
363 list_for_each_entry(spu
, ref
->aff_list
.prev
, aff_list
) {
364 BUG_ON(spu
->node
!= node
);
371 list_for_each_entry_reverse(spu
, ref
->aff_list
.next
, aff_list
) {
372 BUG_ON(spu
->node
!= node
);
384 * affinity_check is called each time a context is going to be scheduled.
385 * It returns the spu ptr on which the context must run.
387 static int has_affinity(struct spu_context
*ctx
)
389 struct spu_gang
*gang
= ctx
->gang
;
391 if (list_empty(&ctx
->aff_list
))
394 mutex_lock(&gang
->aff_mutex
);
395 if (!gang
->aff_ref_spu
) {
396 if (!(gang
->aff_flags
& AFF_MERGED
))
397 aff_merge_remaining_ctxs(gang
);
398 if (!(gang
->aff_flags
& AFF_OFFSETS_SET
))
399 aff_set_offsets(gang
);
400 aff_set_ref_point_location(gang
);
402 mutex_unlock(&gang
->aff_mutex
);
404 return gang
->aff_ref_spu
!= NULL
;
408 * spu_unbind_context - unbind spu context from physical spu
409 * @spu: physical spu to unbind from
410 * @ctx: context to unbind
412 static void spu_unbind_context(struct spu
*spu
, struct spu_context
*ctx
)
414 pr_debug("%s: unbind pid=%d SPU=%d NODE=%d\n", __FUNCTION__
,
415 spu
->pid
, spu
->number
, spu
->node
);
416 spuctx_switch_state(ctx
, SPU_UTIL_SYSTEM
);
418 if (spu
->ctx
->flags
& SPU_CREATE_NOSCHED
)
419 atomic_dec(&cbe_spu_info
[spu
->node
].reserved_spus
);
420 if (!list_empty(&ctx
->aff_list
))
421 if (atomic_dec_and_test(&ctx
->gang
->aff_sched_count
))
422 ctx
->gang
->aff_ref_spu
= NULL
;
423 spu_switch_notify(spu
, NULL
);
424 spu_unmap_mappings(ctx
);
425 spu_save(&ctx
->csa
, spu
);
426 spu
->timestamp
= jiffies
;
427 ctx
->state
= SPU_STATE_SAVED
;
428 spu
->ibox_callback
= NULL
;
429 spu
->wbox_callback
= NULL
;
430 spu
->stop_callback
= NULL
;
431 spu
->mfc_callback
= NULL
;
432 spu
->dma_callback
= NULL
;
433 spu_associate_mm(spu
, NULL
);
436 ctx
->ops
= &spu_backing_ops
;
440 ctx
->stats
.slb_flt
+=
441 (spu
->stats
.slb_flt
- ctx
->stats
.slb_flt_base
);
442 ctx
->stats
.class2_intr
+=
443 (spu
->stats
.class2_intr
- ctx
->stats
.class2_intr_base
);
445 /* This maps the underlying spu state to idle */
446 spuctx_switch_state(ctx
, SPU_UTIL_IDLE_LOADED
);
451 * spu_add_to_rq - add a context to the runqueue
452 * @ctx: context to add
454 static void __spu_add_to_rq(struct spu_context
*ctx
)
457 * Unfortunately this code path can be called from multiple threads
458 * on behalf of a single context due to the way the problem state
459 * mmap support works.
461 * Fortunately we need to wake up all these threads at the same time
462 * and can simply skip the runqueue addition for every but the first
463 * thread getting into this codepath.
465 * It's still quite hacky, and long-term we should proxy all other
466 * threads through the owner thread so that spu_run is in control
467 * of all the scheduling activity for a given context.
469 if (list_empty(&ctx
->rq
)) {
470 list_add_tail(&ctx
->rq
, &spu_prio
->runq
[ctx
->prio
]);
471 set_bit(ctx
->prio
, spu_prio
->bitmap
);
472 if (!spu_prio
->nr_waiting
++)
473 __mod_timer(&spusched_timer
, jiffies
+ SPUSCHED_TICK
);
477 static void __spu_del_from_rq(struct spu_context
*ctx
)
479 int prio
= ctx
->prio
;
481 if (!list_empty(&ctx
->rq
)) {
482 if (!--spu_prio
->nr_waiting
)
483 del_timer(&spusched_timer
);
484 list_del_init(&ctx
->rq
);
486 if (list_empty(&spu_prio
->runq
[prio
]))
487 clear_bit(prio
, spu_prio
->bitmap
);
491 static void spu_prio_wait(struct spu_context
*ctx
)
495 spin_lock(&spu_prio
->runq_lock
);
496 prepare_to_wait_exclusive(&ctx
->stop_wq
, &wait
, TASK_INTERRUPTIBLE
);
497 if (!signal_pending(current
)) {
498 __spu_add_to_rq(ctx
);
499 spin_unlock(&spu_prio
->runq_lock
);
500 mutex_unlock(&ctx
->state_mutex
);
502 mutex_lock(&ctx
->state_mutex
);
503 spin_lock(&spu_prio
->runq_lock
);
504 __spu_del_from_rq(ctx
);
506 spin_unlock(&spu_prio
->runq_lock
);
507 __set_current_state(TASK_RUNNING
);
508 remove_wait_queue(&ctx
->stop_wq
, &wait
);
511 static struct spu
*spu_get_idle(struct spu_context
*ctx
)
516 if (has_affinity(ctx
)) {
517 node
= ctx
->gang
->aff_ref_spu
->node
;
519 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
520 spu
= ctx_location(ctx
->gang
->aff_ref_spu
, ctx
->aff_offset
, node
);
521 if (spu
&& spu
->alloc_state
== SPU_FREE
)
523 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
527 node
= cpu_to_node(raw_smp_processor_id());
528 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
529 node
= (node
< MAX_NUMNODES
) ? node
: 0;
530 if (!node_allowed(ctx
, node
))
533 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
534 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
535 if (spu
->alloc_state
== SPU_FREE
)
538 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
544 spu
->alloc_state
= SPU_USED
;
545 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
546 pr_debug("Got SPU %d %d\n", spu
->number
, spu
->node
);
547 spu_init_channels(spu
);
552 * find_victim - find a lower priority context to preempt
553 * @ctx: canidate context for running
555 * Returns the freed physical spu to run the new context on.
557 static struct spu
*find_victim(struct spu_context
*ctx
)
559 struct spu_context
*victim
= NULL
;
564 * Look for a possible preemption candidate on the local node first.
565 * If there is no candidate look at the other nodes. This isn't
566 * exactly fair, but so far the whole spu schedule tries to keep
567 * a strong node affinity. We might want to fine-tune this in
571 node
= cpu_to_node(raw_smp_processor_id());
572 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
573 node
= (node
< MAX_NUMNODES
) ? node
: 0;
574 if (!node_allowed(ctx
, node
))
577 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
578 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
579 struct spu_context
*tmp
= spu
->ctx
;
581 if (tmp
->prio
> ctx
->prio
&&
582 (!victim
|| tmp
->prio
> victim
->prio
))
585 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
589 * This nests ctx->state_mutex, but we always lock
590 * higher priority contexts before lower priority
591 * ones, so this is safe until we introduce
592 * priority inheritance schemes.
594 if (!mutex_trylock(&victim
->state_mutex
)) {
602 * This race can happen because we've dropped
603 * the active list mutex. No a problem, just
604 * restart the search.
606 mutex_unlock(&victim
->state_mutex
);
611 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
612 cbe_spu_info
[node
].nr_active
--;
613 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
615 spu_unbind_context(spu
, victim
);
616 victim
->stats
.invol_ctx_switch
++;
617 spu
->stats
.invol_ctx_switch
++;
618 mutex_unlock(&victim
->state_mutex
);
620 * We need to break out of the wait loop in spu_run
621 * manually to ensure this context gets put on the
622 * runqueue again ASAP.
624 wake_up(&victim
->stop_wq
);
633 * spu_activate - find a free spu for a context and execute it
634 * @ctx: spu context to schedule
635 * @flags: flags (currently ignored)
637 * Tries to find a free spu to run @ctx. If no free spu is available
638 * add the context to the runqueue so it gets woken up once an spu
641 int spu_activate(struct spu_context
*ctx
, unsigned long flags
)
647 * If there are multiple threads waiting for a single context
648 * only one actually binds the context while the others will
649 * only be able to acquire the state_mutex once the context
650 * already is in runnable state.
655 spu
= spu_get_idle(ctx
);
657 * If this is a realtime thread we try to get it running by
658 * preempting a lower priority thread.
660 if (!spu
&& rt_prio(ctx
->prio
))
661 spu
= find_victim(ctx
);
663 int node
= spu
->node
;
665 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
666 spu_bind_context(spu
, ctx
);
667 cbe_spu_info
[node
].nr_active
++;
668 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
673 } while (!signal_pending(current
));
679 * grab_runnable_context - try to find a runnable context
681 * Remove the highest priority context on the runqueue and return it
682 * to the caller. Returns %NULL if no runnable context was found.
684 static struct spu_context
*grab_runnable_context(int prio
, int node
)
686 struct spu_context
*ctx
;
689 spin_lock(&spu_prio
->runq_lock
);
690 best
= find_first_bit(spu_prio
->bitmap
, prio
);
691 while (best
< prio
) {
692 struct list_head
*rq
= &spu_prio
->runq
[best
];
694 list_for_each_entry(ctx
, rq
, rq
) {
695 /* XXX(hch): check for affinity here aswell */
696 if (__node_allowed(ctx
, node
)) {
697 __spu_del_from_rq(ctx
);
705 spin_unlock(&spu_prio
->runq_lock
);
709 static int __spu_deactivate(struct spu_context
*ctx
, int force
, int max_prio
)
711 struct spu
*spu
= ctx
->spu
;
712 struct spu_context
*new = NULL
;
715 new = grab_runnable_context(max_prio
, spu
->node
);
717 int node
= spu
->node
;
719 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
720 spu_unbind_context(spu
, ctx
);
721 spu
->alloc_state
= SPU_FREE
;
722 cbe_spu_info
[node
].nr_active
--;
723 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
725 ctx
->stats
.vol_ctx_switch
++;
726 spu
->stats
.vol_ctx_switch
++;
729 wake_up(&new->stop_wq
);
738 * spu_deactivate - unbind a context from it's physical spu
739 * @ctx: spu context to unbind
741 * Unbind @ctx from the physical spu it is running on and schedule
742 * the highest priority context to run on the freed physical spu.
744 void spu_deactivate(struct spu_context
*ctx
)
746 __spu_deactivate(ctx
, 1, MAX_PRIO
);
750 * spu_yield - yield a physical spu if others are waiting
751 * @ctx: spu context to yield
753 * Check if there is a higher priority context waiting and if yes
754 * unbind @ctx from the physical spu and schedule the highest
755 * priority context to run on the freed physical spu instead.
757 void spu_yield(struct spu_context
*ctx
)
759 if (!(ctx
->flags
& SPU_CREATE_NOSCHED
)) {
760 mutex_lock(&ctx
->state_mutex
);
761 __spu_deactivate(ctx
, 0, MAX_PRIO
);
762 mutex_unlock(&ctx
->state_mutex
);
766 static noinline
void spusched_tick(struct spu_context
*ctx
)
768 if (ctx
->flags
& SPU_CREATE_NOSCHED
)
770 if (ctx
->policy
== SCHED_FIFO
)
773 if (--ctx
->time_slice
)
777 * Unfortunately list_mutex ranks outside of state_mutex, so
778 * we have to trylock here. If we fail give the context another
779 * tick and try again.
781 if (mutex_trylock(&ctx
->state_mutex
)) {
782 struct spu
*spu
= ctx
->spu
;
783 struct spu_context
*new;
785 new = grab_runnable_context(ctx
->prio
+ 1, spu
->node
);
787 spu_unbind_context(spu
, ctx
);
788 ctx
->stats
.invol_ctx_switch
++;
789 spu
->stats
.invol_ctx_switch
++;
790 spu
->alloc_state
= SPU_FREE
;
791 cbe_spu_info
[spu
->node
].nr_active
--;
792 wake_up(&new->stop_wq
);
794 * We need to break out of the wait loop in
795 * spu_run manually to ensure this context
796 * gets put on the runqueue again ASAP.
798 wake_up(&ctx
->stop_wq
);
800 spu_set_timeslice(ctx
);
801 mutex_unlock(&ctx
->state_mutex
);
808 * count_active_contexts - count nr of active tasks
810 * Return the number of tasks currently running or waiting to run.
812 * Note that we don't take runq_lock / list_mutex here. Reading
813 * a single 32bit value is atomic on powerpc, and we don't care
814 * about memory ordering issues here.
816 static unsigned long count_active_contexts(void)
818 int nr_active
= 0, node
;
820 for (node
= 0; node
< MAX_NUMNODES
; node
++)
821 nr_active
+= cbe_spu_info
[node
].nr_active
;
822 nr_active
+= spu_prio
->nr_waiting
;
828 * spu_calc_load - given tick count, update the avenrun load estimates.
831 * No locking against reading these values from userspace, as for
832 * the CPU loadavg code.
834 static void spu_calc_load(unsigned long ticks
)
836 unsigned long active_tasks
; /* fixed-point */
837 static int count
= LOAD_FREQ
;
841 if (unlikely(count
< 0)) {
842 active_tasks
= count_active_contexts() * FIXED_1
;
844 CALC_LOAD(spu_avenrun
[0], EXP_1
, active_tasks
);
845 CALC_LOAD(spu_avenrun
[1], EXP_5
, active_tasks
);
846 CALC_LOAD(spu_avenrun
[2], EXP_15
, active_tasks
);
852 static void spusched_wake(unsigned long data
)
854 mod_timer(&spusched_timer
, jiffies
+ SPUSCHED_TICK
);
855 wake_up_process(spusched_task
);
856 spu_calc_load(SPUSCHED_TICK
);
859 static int spusched_thread(void *unused
)
864 while (!kthread_should_stop()) {
865 set_current_state(TASK_INTERRUPTIBLE
);
867 for (node
= 0; node
< MAX_NUMNODES
; node
++) {
868 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
869 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
)
871 spusched_tick(spu
->ctx
);
872 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
879 #define LOAD_INT(x) ((x) >> FSHIFT)
880 #define LOAD_FRAC(x) LOAD_INT(((x) & (FIXED_1-1)) * 100)
882 static int show_spu_loadavg(struct seq_file
*s
, void *private)
886 a
= spu_avenrun
[0] + (FIXED_1
/200);
887 b
= spu_avenrun
[1] + (FIXED_1
/200);
888 c
= spu_avenrun
[2] + (FIXED_1
/200);
891 * Note that last_pid doesn't really make much sense for the
892 * SPU loadavg (it even seems very odd on the CPU side..),
893 * but we include it here to have a 100% compatible interface.
895 seq_printf(s
, "%d.%02d %d.%02d %d.%02d %ld/%d %d\n",
896 LOAD_INT(a
), LOAD_FRAC(a
),
897 LOAD_INT(b
), LOAD_FRAC(b
),
898 LOAD_INT(c
), LOAD_FRAC(c
),
899 count_active_contexts(),
900 atomic_read(&nr_spu_contexts
),
901 current
->nsproxy
->pid_ns
->last_pid
);
905 static int spu_loadavg_open(struct inode
*inode
, struct file
*file
)
907 return single_open(file
, show_spu_loadavg
, NULL
);
910 static const struct file_operations spu_loadavg_fops
= {
911 .open
= spu_loadavg_open
,
914 .release
= single_release
,
917 int __init
spu_sched_init(void)
919 struct proc_dir_entry
*entry
;
920 int err
= -ENOMEM
, i
;
922 spu_prio
= kzalloc(sizeof(struct spu_prio_array
), GFP_KERNEL
);
926 for (i
= 0; i
< MAX_PRIO
; i
++) {
927 INIT_LIST_HEAD(&spu_prio
->runq
[i
]);
928 __clear_bit(i
, spu_prio
->bitmap
);
930 for (i
= 0; i
< MAX_NUMNODES
; i
++) {
931 mutex_init(&cbe_spu_info
[i
].list_mutex
);
932 INIT_LIST_HEAD(&cbe_spu_info
[i
].spus
);
934 spin_lock_init(&spu_prio
->runq_lock
);
936 setup_timer(&spusched_timer
, spusched_wake
, 0);
938 spusched_task
= kthread_run(spusched_thread
, NULL
, "spusched");
939 if (IS_ERR(spusched_task
)) {
940 err
= PTR_ERR(spusched_task
);
941 goto out_free_spu_prio
;
944 entry
= create_proc_entry("spu_loadavg", 0, NULL
);
946 goto out_stop_kthread
;
947 entry
->proc_fops
= &spu_loadavg_fops
;
949 pr_debug("spusched: tick: %d, min ticks: %d, default ticks: %d\n",
950 SPUSCHED_TICK
, MIN_SPU_TIMESLICE
, DEF_SPU_TIMESLICE
);
954 kthread_stop(spusched_task
);
961 void spu_sched_exit(void)
966 remove_proc_entry("spu_loadavg", NULL
);
968 del_timer_sync(&spusched_timer
);
969 kthread_stop(spusched_task
);
971 for (node
= 0; node
< MAX_NUMNODES
; node
++) {
972 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
973 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
)
974 if (spu
->alloc_state
!= SPU_FREE
)
975 spu
->alloc_state
= SPU_FREE
;
976 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);