2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/time.h>
13 #include <linux/delay.h>
14 #include <linux/init.h>
15 #include <linux/smp.h>
16 #include <linux/types.h>
17 #include <linux/timex.h>
18 #include <linux/notifier.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/rcupdate.h>
21 #include <linux/posix-timers.h>
23 #include <asm/s390_ext.h>
24 #include <asm/timer.h>
25 #include <asm/irq_regs.h>
27 static ext_int_info_t ext_int_info_timer
;
28 static DEFINE_PER_CPU(struct vtimer_queue
, virt_cpu_timer
);
31 * Update process times based on virtual cpu times stored by entry.S
32 * to the lowcore fields user_timer, system_timer & steal_clock.
34 void account_process_tick(struct task_struct
*tsk
, int user_tick
)
40 timer
= S390_lowcore
.last_update_timer
;
41 clock
= S390_lowcore
.last_update_clock
;
42 asm volatile (" STPT %0\n" /* Store current cpu timer value */
43 " STCK %1" /* Store current tod clock value */
44 : "=m" (S390_lowcore
.last_update_timer
),
45 "=m" (S390_lowcore
.last_update_clock
) );
46 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
47 S390_lowcore
.steal_clock
+= S390_lowcore
.last_update_clock
- clock
;
49 cputime
= S390_lowcore
.user_timer
>> 12;
50 rcu_user_flag
= cputime
!= 0;
51 S390_lowcore
.user_timer
-= cputime
<< 12;
52 S390_lowcore
.steal_clock
-= cputime
<< 12;
53 account_user_time(tsk
, cputime
);
55 cputime
= S390_lowcore
.system_timer
>> 12;
56 S390_lowcore
.system_timer
-= cputime
<< 12;
57 S390_lowcore
.steal_clock
-= cputime
<< 12;
58 account_system_time(tsk
, HARDIRQ_OFFSET
, cputime
);
60 cputime
= S390_lowcore
.steal_clock
;
61 if ((__s64
) cputime
> 0) {
63 S390_lowcore
.steal_clock
-= cputime
<< 12;
64 account_steal_time(tsk
, cputime
);
69 * Update process times based on virtual cpu times stored by entry.S
70 * to the lowcore fields user_timer, system_timer & steal_clock.
72 void account_vtime(struct task_struct
*tsk
)
77 timer
= S390_lowcore
.last_update_timer
;
78 asm volatile (" STPT %0" /* Store current cpu timer value */
79 : "=m" (S390_lowcore
.last_update_timer
) );
80 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
82 cputime
= S390_lowcore
.user_timer
>> 12;
83 S390_lowcore
.user_timer
-= cputime
<< 12;
84 S390_lowcore
.steal_clock
-= cputime
<< 12;
85 account_user_time(tsk
, cputime
);
87 cputime
= S390_lowcore
.system_timer
>> 12;
88 S390_lowcore
.system_timer
-= cputime
<< 12;
89 S390_lowcore
.steal_clock
-= cputime
<< 12;
90 account_system_time(tsk
, 0, cputime
);
94 * Update process times based on virtual cpu times stored by entry.S
95 * to the lowcore fields user_timer, system_timer & steal_clock.
97 void account_system_vtime(struct task_struct
*tsk
)
102 timer
= S390_lowcore
.last_update_timer
;
103 asm volatile (" STPT %0" /* Store current cpu timer value */
104 : "=m" (S390_lowcore
.last_update_timer
) );
105 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
107 cputime
= S390_lowcore
.system_timer
>> 12;
108 S390_lowcore
.system_timer
-= cputime
<< 12;
109 S390_lowcore
.steal_clock
-= cputime
<< 12;
110 account_system_time(tsk
, 0, cputime
);
112 EXPORT_SYMBOL_GPL(account_system_vtime
);
114 static inline void set_vtimer(__u64 expires
)
118 asm volatile (" STPT %0\n" /* Store current cpu timer value */
119 " SPT %1" /* Set new value immediatly afterwards */
120 : "=m" (timer
) : "m" (expires
) );
121 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- timer
;
122 S390_lowcore
.last_update_timer
= expires
;
124 /* store expire time for this CPU timer */
125 __get_cpu_var(virt_cpu_timer
).to_expire
= expires
;
128 void vtime_start_cpu_timer(void)
130 struct vtimer_queue
*vt_list
;
132 vt_list
= &__get_cpu_var(virt_cpu_timer
);
134 /* CPU timer interrupt is pending, don't reprogramm it */
135 if (vt_list
->idle
& 1LL<<63)
138 if (!list_empty(&vt_list
->list
))
139 set_vtimer(vt_list
->idle
);
142 void vtime_stop_cpu_timer(void)
144 struct vtimer_queue
*vt_list
;
146 vt_list
= &__get_cpu_var(virt_cpu_timer
);
149 if (list_empty(&vt_list
->list
)) {
150 vt_list
->idle
= VTIMER_MAX_SLICE
;
154 /* store the actual expire value */
155 asm volatile ("STPT %0" : "=m" (vt_list
->idle
));
158 * If the CPU timer is negative we don't reprogramm
159 * it because we will get instantly an interrupt.
161 if (vt_list
->idle
& 1LL<<63)
164 vt_list
->offset
+= vt_list
->to_expire
- vt_list
->idle
;
167 * We cannot halt the CPU timer, we just write a value that
168 * nearly never expires (only after 71 years) and re-write
169 * the stored expire value if we continue the timer
172 set_vtimer(VTIMER_MAX_SLICE
);
176 * Sorted add to a list. List is linear searched until first bigger
179 static void list_add_sorted(struct vtimer_list
*timer
, struct list_head
*head
)
181 struct vtimer_list
*event
;
183 list_for_each_entry(event
, head
, entry
) {
184 if (event
->expires
> timer
->expires
) {
185 list_add_tail(&timer
->entry
, &event
->entry
);
189 list_add_tail(&timer
->entry
, head
);
193 * Do the callback functions of expired vtimer events.
194 * Called from within the interrupt handler.
196 static void do_callbacks(struct list_head
*cb_list
)
198 struct vtimer_queue
*vt_list
;
199 struct vtimer_list
*event
, *tmp
;
200 void (*fn
)(unsigned long);
203 if (list_empty(cb_list
))
206 vt_list
= &__get_cpu_var(virt_cpu_timer
);
208 list_for_each_entry_safe(event
, tmp
, cb_list
, entry
) {
209 fn
= event
->function
;
213 if (!event
->interval
)
214 /* delete one shot timer */
215 list_del_init(&event
->entry
);
217 /* move interval timer back to list */
218 spin_lock(&vt_list
->lock
);
219 list_del_init(&event
->entry
);
220 list_add_sorted(event
, &vt_list
->list
);
221 spin_unlock(&vt_list
->lock
);
227 * Handler for the virtual CPU timer.
229 static void do_cpu_timer_interrupt(__u16 error_code
)
232 struct vtimer_queue
*vt_list
;
233 struct vtimer_list
*event
, *tmp
;
234 struct list_head
*ptr
;
235 /* the callback queue */
236 struct list_head cb_list
;
238 INIT_LIST_HEAD(&cb_list
);
239 vt_list
= &__get_cpu_var(virt_cpu_timer
);
241 /* walk timer list, fire all expired events */
242 spin_lock(&vt_list
->lock
);
244 if (vt_list
->to_expire
< VTIMER_MAX_SLICE
)
245 vt_list
->offset
+= vt_list
->to_expire
;
247 list_for_each_entry_safe(event
, tmp
, &vt_list
->list
, entry
) {
248 if (event
->expires
> vt_list
->offset
)
249 /* found first unexpired event, leave */
252 /* re-charge interval timer, we have to add the offset */
254 event
->expires
= event
->interval
+ vt_list
->offset
;
256 /* move expired timer to the callback queue */
257 list_move_tail(&event
->entry
, &cb_list
);
259 spin_unlock(&vt_list
->lock
);
260 do_callbacks(&cb_list
);
262 /* next event is first in list */
263 spin_lock(&vt_list
->lock
);
264 if (!list_empty(&vt_list
->list
)) {
265 ptr
= vt_list
->list
.next
;
266 event
= list_entry(ptr
, struct vtimer_list
, entry
);
267 next
= event
->expires
- vt_list
->offset
;
269 /* add the expired time from this interrupt handler
270 * and the callback functions
272 asm volatile ("STPT %0" : "=m" (delta
));
273 delta
= 0xffffffffffffffffLL
- delta
+ 1;
274 vt_list
->offset
+= delta
;
278 next
= VTIMER_MAX_SLICE
;
280 spin_unlock(&vt_list
->lock
);
284 void init_virt_timer(struct vtimer_list
*timer
)
286 timer
->function
= NULL
;
287 INIT_LIST_HEAD(&timer
->entry
);
288 spin_lock_init(&timer
->lock
);
290 EXPORT_SYMBOL(init_virt_timer
);
292 static inline int vtimer_pending(struct vtimer_list
*timer
)
294 return (!list_empty(&timer
->entry
));
298 * this function should only run on the specified CPU
300 static void internal_add_vtimer(struct vtimer_list
*timer
)
304 struct vtimer_list
*event
;
305 struct vtimer_queue
*vt_list
;
307 vt_list
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
308 spin_lock_irqsave(&vt_list
->lock
, flags
);
310 BUG_ON(timer
->cpu
!= smp_processor_id());
312 /* if list is empty we only have to set the timer */
313 if (list_empty(&vt_list
->list
)) {
314 /* reset the offset, this may happen if the last timer was
315 * just deleted by mod_virt_timer and the interrupt
316 * didn't happen until here
323 asm volatile ("STPT %0" : "=m" (done
));
325 /* calculate completed work */
326 done
= vt_list
->to_expire
- done
+ vt_list
->offset
;
329 list_for_each_entry(event
, &vt_list
->list
, entry
)
330 event
->expires
-= done
;
333 list_add_sorted(timer
, &vt_list
->list
);
335 /* get first element, which is the next vtimer slice */
336 event
= list_entry(vt_list
->list
.next
, struct vtimer_list
, entry
);
338 set_vtimer(event
->expires
);
339 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
340 /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
344 static inline void prepare_vtimer(struct vtimer_list
*timer
)
346 BUG_ON(!timer
->function
);
347 BUG_ON(!timer
->expires
|| timer
->expires
> VTIMER_MAX_SLICE
);
348 BUG_ON(vtimer_pending(timer
));
349 timer
->cpu
= get_cpu();
353 * add_virt_timer - add an oneshot virtual CPU timer
355 void add_virt_timer(void *new)
357 struct vtimer_list
*timer
;
359 timer
= (struct vtimer_list
*)new;
360 prepare_vtimer(timer
);
362 internal_add_vtimer(timer
);
364 EXPORT_SYMBOL(add_virt_timer
);
367 * add_virt_timer_int - add an interval virtual CPU timer
369 void add_virt_timer_periodic(void *new)
371 struct vtimer_list
*timer
;
373 timer
= (struct vtimer_list
*)new;
374 prepare_vtimer(timer
);
375 timer
->interval
= timer
->expires
;
376 internal_add_vtimer(timer
);
378 EXPORT_SYMBOL(add_virt_timer_periodic
);
381 * If we change a pending timer the function must be called on the CPU
382 * where the timer is running on, e.g. by smp_call_function_single()
384 * The original mod_timer adds the timer if it is not pending. For compatibility
385 * we do the same. The timer will be added on the current CPU as a oneshot timer.
387 * returns whether it has modified a pending timer (1) or not (0)
389 int mod_virt_timer(struct vtimer_list
*timer
, __u64 expires
)
391 struct vtimer_queue
*vt_list
;
395 BUG_ON(!timer
->function
);
396 BUG_ON(!expires
|| expires
> VTIMER_MAX_SLICE
);
399 * This is a common optimization triggered by the
400 * networking code - if the timer is re-modified
401 * to be the same thing then just return:
403 if (timer
->expires
== expires
&& vtimer_pending(timer
))
407 vt_list
= &per_cpu(virt_cpu_timer
, cpu
);
409 /* check if we run on the right CPU */
410 BUG_ON(timer
->cpu
!= cpu
);
412 /* disable interrupts before test if timer is pending */
413 spin_lock_irqsave(&vt_list
->lock
, flags
);
415 /* if timer isn't pending add it on the current CPU */
416 if (!vtimer_pending(timer
)) {
417 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
418 /* we do not activate an interval timer with mod_virt_timer */
420 timer
->expires
= expires
;
422 internal_add_vtimer(timer
);
426 list_del_init(&timer
->entry
);
427 timer
->expires
= expires
;
429 /* also change the interval if we have an interval timer */
431 timer
->interval
= expires
;
433 /* the timer can't expire anymore so we can release the lock */
434 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
435 internal_add_vtimer(timer
);
438 EXPORT_SYMBOL(mod_virt_timer
);
441 * delete a virtual timer
443 * returns whether the deleted timer was pending (1) or not (0)
445 int del_virt_timer(struct vtimer_list
*timer
)
448 struct vtimer_queue
*vt_list
;
450 /* check if timer is pending */
451 if (!vtimer_pending(timer
))
454 vt_list
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
455 spin_lock_irqsave(&vt_list
->lock
, flags
);
457 /* we don't interrupt a running timer, just let it expire! */
458 list_del_init(&timer
->entry
);
460 /* last timer removed */
461 if (list_empty(&vt_list
->list
)) {
462 vt_list
->to_expire
= 0;
466 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
469 EXPORT_SYMBOL(del_virt_timer
);
472 * Start the virtual CPU timer on the current CPU.
474 void init_cpu_vtimer(void)
476 struct vtimer_queue
*vt_list
;
478 /* kick the virtual timer */
479 S390_lowcore
.exit_timer
= VTIMER_MAX_SLICE
;
480 S390_lowcore
.last_update_timer
= VTIMER_MAX_SLICE
;
481 asm volatile ("SPT %0" : : "m" (S390_lowcore
.last_update_timer
));
482 asm volatile ("STCK %0" : "=m" (S390_lowcore
.last_update_clock
));
484 /* enable cpu timer interrupts */
487 vt_list
= &__get_cpu_var(virt_cpu_timer
);
488 INIT_LIST_HEAD(&vt_list
->list
);
489 spin_lock_init(&vt_list
->lock
);
490 vt_list
->to_expire
= 0;
496 void __init
vtime_init(void)
498 /* request the cpu timer external interrupt */
499 if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt
,
500 &ext_int_info_timer
) != 0)
501 panic("Couldn't request external interrupt 0x1005");
503 /* Enable cpu timer interrupts on the boot cpu. */