2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/time.h>
13 #include <linux/delay.h>
14 #include <linux/init.h>
15 #include <linux/smp.h>
16 #include <linux/types.h>
17 #include <linux/timex.h>
18 #include <linux/notifier.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/rcupdate.h>
21 #include <linux/posix-timers.h>
23 #include <asm/s390_ext.h>
24 #include <asm/timer.h>
25 #include <asm/irq_regs.h>
26 #include <asm/cputime.h>
28 static DEFINE_PER_CPU(struct vtimer_queue
, virt_cpu_timer
);
30 DEFINE_PER_CPU(struct s390_idle_data
, s390_idle
);
32 static inline __u64
get_vtimer(void)
36 asm volatile("STPT %0" : "=m" (timer
));
40 static inline void set_vtimer(__u64 expires
)
44 asm volatile (" STPT %0\n" /* Store current cpu timer value */
45 " SPT %1" /* Set new value immediatly afterwards */
46 : "=m" (timer
) : "m" (expires
) );
47 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- timer
;
48 S390_lowcore
.last_update_timer
= expires
;
52 * Update process times based on virtual cpu times stored by entry.S
53 * to the lowcore fields user_timer, system_timer & steal_clock.
55 static void do_account_vtime(struct task_struct
*tsk
, int hardirq_offset
)
57 struct thread_info
*ti
= task_thread_info(tsk
);
58 __u64 timer
, clock
, user
, system
, steal
;
60 timer
= S390_lowcore
.last_update_timer
;
61 clock
= S390_lowcore
.last_update_clock
;
62 asm volatile (" STPT %0\n" /* Store current cpu timer value */
63 " STCK %1" /* Store current tod clock value */
64 : "=m" (S390_lowcore
.last_update_timer
),
65 "=m" (S390_lowcore
.last_update_clock
) );
66 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
67 S390_lowcore
.steal_timer
+= S390_lowcore
.last_update_clock
- clock
;
69 user
= S390_lowcore
.user_timer
- ti
->user_timer
;
70 S390_lowcore
.steal_timer
-= user
;
71 ti
->user_timer
= S390_lowcore
.user_timer
;
72 account_user_time(tsk
, user
, user
);
74 system
= S390_lowcore
.system_timer
- ti
->system_timer
;
75 S390_lowcore
.steal_timer
-= system
;
76 ti
->system_timer
= S390_lowcore
.system_timer
;
77 account_system_time(tsk
, hardirq_offset
, system
, system
);
79 steal
= S390_lowcore
.steal_timer
;
80 if ((s64
) steal
> 0) {
81 S390_lowcore
.steal_timer
= 0;
82 account_steal_time(steal
);
86 void account_vtime(struct task_struct
*prev
, struct task_struct
*next
)
88 struct thread_info
*ti
;
90 do_account_vtime(prev
, 0);
91 ti
= task_thread_info(prev
);
92 ti
->user_timer
= S390_lowcore
.user_timer
;
93 ti
->system_timer
= S390_lowcore
.system_timer
;
94 ti
= task_thread_info(next
);
95 S390_lowcore
.user_timer
= ti
->user_timer
;
96 S390_lowcore
.system_timer
= ti
->system_timer
;
99 void account_process_tick(struct task_struct
*tsk
, int user_tick
)
101 do_account_vtime(tsk
, HARDIRQ_OFFSET
);
105 * Update process times based on virtual cpu times stored by entry.S
106 * to the lowcore fields user_timer, system_timer & steal_clock.
108 void account_system_vtime(struct task_struct
*tsk
)
110 struct thread_info
*ti
= task_thread_info(tsk
);
113 timer
= S390_lowcore
.last_update_timer
;
114 S390_lowcore
.last_update_timer
= get_vtimer();
115 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
117 system
= S390_lowcore
.system_timer
- ti
->system_timer
;
118 S390_lowcore
.steal_timer
-= system
;
119 ti
->system_timer
= S390_lowcore
.system_timer
;
120 account_system_time(tsk
, 0, system
, system
);
122 EXPORT_SYMBOL_GPL(account_system_vtime
);
124 void vtime_start_cpu(__u64 int_clock
, __u64 enter_timer
)
126 struct s390_idle_data
*idle
= &__get_cpu_var(s390_idle
);
127 struct vtimer_queue
*vq
= &__get_cpu_var(virt_cpu_timer
);
128 __u64 idle_time
, expires
;
130 if (idle
->idle_enter
== 0ULL)
133 /* Account time spent with enabled wait psw loaded as idle time. */
134 idle_time
= int_clock
- idle
->idle_enter
;
135 account_idle_time(idle_time
);
136 S390_lowcore
.steal_timer
+=
137 idle
->idle_enter
- S390_lowcore
.last_update_clock
;
138 S390_lowcore
.last_update_clock
= int_clock
;
140 /* Account system time spent going idle. */
141 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- vq
->idle
;
142 S390_lowcore
.last_update_timer
= enter_timer
;
144 /* Restart vtime CPU timer */
146 /* Program old expire value but first save progress. */
147 expires
= vq
->idle
- enter_timer
;
148 expires
+= get_vtimer();
151 /* Don't account the CPU timer delta while the cpu was idle. */
152 vq
->elapsed
-= vq
->idle
- enter_timer
;
157 idle
->idle_time
+= idle_time
;
158 idle
->idle_enter
= 0ULL;
164 void vtime_stop_cpu(void)
166 struct s390_idle_data
*idle
= &__get_cpu_var(s390_idle
);
167 struct vtimer_queue
*vq
= &__get_cpu_var(virt_cpu_timer
);
170 /* Wait for external, I/O or machine check interrupt. */
171 psw
.mask
= psw_kernel_bits
| PSW_MASK_WAIT
| PSW_MASK_IO
| PSW_MASK_EXT
;
173 idle
->nohz_delay
= 0;
175 /* Check if the CPU timer needs to be reprogrammed. */
177 __u64 vmax
= VTIMER_MAX_SLICE
;
179 * The inline assembly is equivalent to
180 * vq->idle = get_cpu_timer();
181 * set_cpu_timer(VTIMER_MAX_SLICE);
182 * idle->idle_enter = get_clock();
183 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
184 * PSW_MASK_IO | PSW_MASK_EXT);
185 * The difference is that the inline assembly makes sure that
186 * the last three instruction are stpt, stck and lpsw in that
187 * order. This is done to increase the precision.
194 #else /* CONFIG_64BIT */
197 #endif /* CONFIG_64BIT */
203 #else /* CONFIG_64BIT */
205 #endif /* CONFIG_64BIT */
207 : "=m" (idle
->idle_enter
), "=m" (vq
->idle
)
208 : "a" (&psw
), "a" (&idle
->idle_enter
),
209 "a" (&vq
->idle
), "a" (&vmax
), "m" (vmax
), "m" (psw
)
210 : "memory", "cc", "1");
213 * The inline assembly is equivalent to
214 * vq->idle = get_cpu_timer();
215 * idle->idle_enter = get_clock();
216 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
217 * PSW_MASK_IO | PSW_MASK_EXT);
218 * The difference is that the inline assembly makes sure that
219 * the last three instruction are stpt, stck and lpsw in that
220 * order. This is done to increase the precision.
227 #else /* CONFIG_64BIT */
230 #endif /* CONFIG_64BIT */
235 #else /* CONFIG_64BIT */
237 #endif /* CONFIG_64BIT */
239 : "=m" (idle
->idle_enter
), "=m" (vq
->idle
)
240 : "a" (&psw
), "a" (&idle
->idle_enter
),
241 "a" (&vq
->idle
), "m" (psw
)
242 : "memory", "cc", "1");
246 cputime64_t
s390_get_idle_time(int cpu
)
248 struct s390_idle_data
*idle
;
249 unsigned long long now
, idle_time
, idle_enter
;
250 unsigned int sequence
;
252 idle
= &per_cpu(s390_idle
, cpu
);
256 sequence
= idle
->sequence
;
261 idle_enter
= idle
->idle_enter
;
262 if (idle_enter
!= 0ULL && idle_enter
< now
)
263 idle_time
= now
- idle_enter
;
265 if (idle
->sequence
!= sequence
)
271 * Sorted add to a list. List is linear searched until first bigger
274 static void list_add_sorted(struct vtimer_list
*timer
, struct list_head
*head
)
276 struct vtimer_list
*event
;
278 list_for_each_entry(event
, head
, entry
) {
279 if (event
->expires
> timer
->expires
) {
280 list_add_tail(&timer
->entry
, &event
->entry
);
284 list_add_tail(&timer
->entry
, head
);
288 * Do the callback functions of expired vtimer events.
289 * Called from within the interrupt handler.
291 static void do_callbacks(struct list_head
*cb_list
)
293 struct vtimer_queue
*vq
;
294 struct vtimer_list
*event
, *tmp
;
296 if (list_empty(cb_list
))
299 vq
= &__get_cpu_var(virt_cpu_timer
);
301 list_for_each_entry_safe(event
, tmp
, cb_list
, entry
) {
302 list_del_init(&event
->entry
);
303 (event
->function
)(event
->data
);
304 if (event
->interval
) {
305 /* Recharge interval timer */
306 event
->expires
= event
->interval
+ vq
->elapsed
;
307 spin_lock(&vq
->lock
);
308 list_add_sorted(event
, &vq
->list
);
309 spin_unlock(&vq
->lock
);
315 * Handler for the virtual CPU timer.
317 static void do_cpu_timer_interrupt(__u16 error_code
)
319 struct vtimer_queue
*vq
;
320 struct vtimer_list
*event
, *tmp
;
321 struct list_head cb_list
; /* the callback queue */
324 INIT_LIST_HEAD(&cb_list
);
325 vq
= &__get_cpu_var(virt_cpu_timer
);
327 /* walk timer list, fire all expired events */
328 spin_lock(&vq
->lock
);
330 elapsed
= vq
->elapsed
+ (vq
->timer
- S390_lowcore
.async_enter_timer
);
331 BUG_ON((s64
) elapsed
< 0);
333 list_for_each_entry_safe(event
, tmp
, &vq
->list
, entry
) {
334 if (event
->expires
< elapsed
)
335 /* move expired timer to the callback queue */
336 list_move_tail(&event
->entry
, &cb_list
);
338 event
->expires
-= elapsed
;
340 spin_unlock(&vq
->lock
);
342 vq
->do_spt
= list_empty(&cb_list
);
343 do_callbacks(&cb_list
);
345 /* next event is first in list */
346 next
= VTIMER_MAX_SLICE
;
347 spin_lock(&vq
->lock
);
348 if (!list_empty(&vq
->list
)) {
349 event
= list_first_entry(&vq
->list
, struct vtimer_list
, entry
);
350 next
= event
->expires
;
353 spin_unlock(&vq
->lock
);
355 * To improve precision add the time spent by the
356 * interrupt handler to the elapsed time.
357 * Note: CPU timer counts down and we got an interrupt,
358 * the current content is negative
360 elapsed
= S390_lowcore
.async_enter_timer
- get_vtimer();
361 set_vtimer(next
- elapsed
);
362 vq
->timer
= next
- elapsed
;
363 vq
->elapsed
= elapsed
;
366 void init_virt_timer(struct vtimer_list
*timer
)
368 timer
->function
= NULL
;
369 INIT_LIST_HEAD(&timer
->entry
);
371 EXPORT_SYMBOL(init_virt_timer
);
373 static inline int vtimer_pending(struct vtimer_list
*timer
)
375 return (!list_empty(&timer
->entry
));
379 * this function should only run on the specified CPU
381 static void internal_add_vtimer(struct vtimer_list
*timer
)
383 struct vtimer_queue
*vq
;
387 vq
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
388 spin_lock_irqsave(&vq
->lock
, flags
);
390 BUG_ON(timer
->cpu
!= smp_processor_id());
392 if (list_empty(&vq
->list
)) {
393 /* First timer on this cpu, just program it. */
394 list_add(&timer
->entry
, &vq
->list
);
395 set_vtimer(timer
->expires
);
396 vq
->timer
= timer
->expires
;
399 /* Check progress of old timers. */
400 expires
= timer
->expires
;
402 if (likely((s64
) expires
< (s64
) left
)) {
403 /* The new timer expires before the current timer. */
405 vq
->elapsed
+= vq
->timer
- left
;
408 vq
->elapsed
+= vq
->timer
- left
;
411 /* Insert new timer into per cpu list. */
412 timer
->expires
+= vq
->elapsed
;
413 list_add_sorted(timer
, &vq
->list
);
416 spin_unlock_irqrestore(&vq
->lock
, flags
);
417 /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
421 static inline void prepare_vtimer(struct vtimer_list
*timer
)
423 BUG_ON(!timer
->function
);
424 BUG_ON(!timer
->expires
|| timer
->expires
> VTIMER_MAX_SLICE
);
425 BUG_ON(vtimer_pending(timer
));
426 timer
->cpu
= get_cpu();
430 * add_virt_timer - add an oneshot virtual CPU timer
432 void add_virt_timer(void *new)
434 struct vtimer_list
*timer
;
436 timer
= (struct vtimer_list
*)new;
437 prepare_vtimer(timer
);
439 internal_add_vtimer(timer
);
441 EXPORT_SYMBOL(add_virt_timer
);
444 * add_virt_timer_int - add an interval virtual CPU timer
446 void add_virt_timer_periodic(void *new)
448 struct vtimer_list
*timer
;
450 timer
= (struct vtimer_list
*)new;
451 prepare_vtimer(timer
);
452 timer
->interval
= timer
->expires
;
453 internal_add_vtimer(timer
);
455 EXPORT_SYMBOL(add_virt_timer_periodic
);
457 int __mod_vtimer(struct vtimer_list
*timer
, __u64 expires
, int periodic
)
459 struct vtimer_queue
*vq
;
463 BUG_ON(!timer
->function
);
464 BUG_ON(!expires
|| expires
> VTIMER_MAX_SLICE
);
466 if (timer
->expires
== expires
&& vtimer_pending(timer
))
470 vq
= &per_cpu(virt_cpu_timer
, cpu
);
472 /* disable interrupts before test if timer is pending */
473 spin_lock_irqsave(&vq
->lock
, flags
);
475 /* if timer isn't pending add it on the current CPU */
476 if (!vtimer_pending(timer
)) {
477 spin_unlock_irqrestore(&vq
->lock
, flags
);
480 timer
->interval
= expires
;
483 timer
->expires
= expires
;
485 internal_add_vtimer(timer
);
489 /* check if we run on the right CPU */
490 BUG_ON(timer
->cpu
!= cpu
);
492 list_del_init(&timer
->entry
);
493 timer
->expires
= expires
;
495 timer
->interval
= expires
;
497 /* the timer can't expire anymore so we can release the lock */
498 spin_unlock_irqrestore(&vq
->lock
, flags
);
499 internal_add_vtimer(timer
);
504 * If we change a pending timer the function must be called on the CPU
505 * where the timer is running on.
507 * returns whether it has modified a pending timer (1) or not (0)
509 int mod_virt_timer(struct vtimer_list
*timer
, __u64 expires
)
511 return __mod_vtimer(timer
, expires
, 0);
513 EXPORT_SYMBOL(mod_virt_timer
);
516 * If we change a pending timer the function must be called on the CPU
517 * where the timer is running on.
519 * returns whether it has modified a pending timer (1) or not (0)
521 int mod_virt_timer_periodic(struct vtimer_list
*timer
, __u64 expires
)
523 return __mod_vtimer(timer
, expires
, 1);
525 EXPORT_SYMBOL(mod_virt_timer_periodic
);
528 * delete a virtual timer
530 * returns whether the deleted timer was pending (1) or not (0)
532 int del_virt_timer(struct vtimer_list
*timer
)
535 struct vtimer_queue
*vq
;
537 /* check if timer is pending */
538 if (!vtimer_pending(timer
))
541 vq
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
542 spin_lock_irqsave(&vq
->lock
, flags
);
544 /* we don't interrupt a running timer, just let it expire! */
545 list_del_init(&timer
->entry
);
547 spin_unlock_irqrestore(&vq
->lock
, flags
);
550 EXPORT_SYMBOL(del_virt_timer
);
553 * Start the virtual CPU timer on the current CPU.
555 void init_cpu_vtimer(void)
557 struct vtimer_queue
*vq
;
559 /* initialize per cpu vtimer structure */
560 vq
= &__get_cpu_var(virt_cpu_timer
);
561 INIT_LIST_HEAD(&vq
->list
);
562 spin_lock_init(&vq
->lock
);
564 /* enable cpu timer interrupts */
568 void __init
vtime_init(void)
570 /* request the cpu timer external interrupt */
571 if (register_external_interrupt(0x1005, do_cpu_timer_interrupt
))
572 panic("Couldn't request external interrupt 0x1005");
574 /* Enable cpu timer interrupts on the boot cpu. */
This page took 0.07046 seconds and 5 git commands to generate.