arch/s390/kernel/vtime.c

   1 /*
   2  *    Virtual cpu timer based timer functions.
   3  *
   4  *    Copyright IBM Corp. 2004, 2012
   5  *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
   6  */
   7
   8 #include <linux/kernel_stat.h>
   9 #include <linux/notifier.h>
  10 #include <linux/kprobes.h>
  11 #include <linux/export.h>
  12 #include <linux/kernel.h>
  13 #include <linux/timex.h>
  14 #include <linux/types.h>
  15 #include <linux/time.h>
  16 #include <linux/cpu.h>
  17 #include <linux/smp.h>
  18
  19 #include <asm/irq_regs.h>
  20 #include <asm/cputime.h>
  21 #include <asm/vtimer.h>
  22 #include <asm/irq.h>
  23 #include "entry.h"
  24
  25 static void virt_timer_expire(void);
  26
  27 DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
  28
  29 static LIST_HEAD(virt_timer_list);
  30 static DEFINE_SPINLOCK(virt_timer_lock);
  31 static atomic64_t virt_timer_current;
  32 static atomic64_t virt_timer_elapsed;
  33
  34 static inline u64 get_vtimer(void)
  35 {
  36         u64 timer;
  37
  38         asm volatile("stpt %0" : "=m" (timer));
  39         return timer;
  40 }
  41
  42 static inline void set_vtimer(u64 expires)
  43 {
  44         u64 timer;
  45
  46         asm volatile(
  47                 "       stpt    %0\n"   /* Store current cpu timer value */
  48                 "       spt     %1"     /* Set new value imm. afterwards */
  49                 : "=m" (timer) : "m" (expires));
  50         S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
  51         S390_lowcore.last_update_timer = expires;
  52 }
  53
  54 static inline int virt_timer_forward(u64 elapsed)
  55 {
  56         BUG_ON(!irqs_disabled());
  57
  58         if (list_empty(&virt_timer_list))
  59                 return 0;
  60         elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
  61         return elapsed >= atomic64_read(&virt_timer_current);
  62 }
  63
  64 /*
  65  * Update process times based on virtual cpu times stored by entry.S
  66  * to the lowcore fields user_timer, system_timer & steal_clock.
  67  */
  68 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
  69 {
  70         struct thread_info *ti = task_thread_info(tsk);
  71         u64 timer, clock, user, system, steal;
  72
  73         timer = S390_lowcore.last_update_timer;
  74         clock = S390_lowcore.last_update_clock;
  75         asm volatile(
  76                 "       stpt    %0\n"   /* Store current cpu timer value */
  77                 "       stck    %1"     /* Store current tod clock value */
  78                 : "=m" (S390_lowcore.last_update_timer),
  79                   "=m" (S390_lowcore.last_update_clock));
  80         S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
  81         S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
  82
  83         user = S390_lowcore.user_timer - ti->user_timer;
  84         S390_lowcore.steal_timer -= user;
  85         ti->user_timer = S390_lowcore.user_timer;
  86         account_user_time(tsk, user, user);
  87
  88         system = S390_lowcore.system_timer - ti->system_timer;
  89         S390_lowcore.steal_timer -= system;
  90         ti->system_timer = S390_lowcore.system_timer;
  91         account_system_time(tsk, hardirq_offset, system, system);
  92
  93         steal = S390_lowcore.steal_timer;
  94         if ((s64) steal > 0) {
  95                 S390_lowcore.steal_timer = 0;
  96                 account_steal_time(steal);
  97         }
  98
  99         return virt_timer_forward(user + system);
 100 }
 101
 102 void vtime_task_switch(struct task_struct *prev)
 103 {
 104         struct thread_info *ti;
 105
 106         do_account_vtime(prev, 0);
 107         ti = task_thread_info(prev);
 108         ti->user_timer = S390_lowcore.user_timer;
 109         ti->system_timer = S390_lowcore.system_timer;
 110         ti = task_thread_info(current);
 111         S390_lowcore.user_timer = ti->user_timer;
 112         S390_lowcore.system_timer = ti->system_timer;
 113 }
 114
 115 /*
 116  * In s390, accounting pending user time also implies
 117  * accounting system time in order to correctly compute
 118  * the stolen time accounting.
 119  */
 120 void vtime_account_user(struct task_struct *tsk)
 121 {
 122         if (do_account_vtime(tsk, HARDIRQ_OFFSET))
 123                 virt_timer_expire();
 124 }
 125
 126 /*
 127  * Update process times based on virtual cpu times stored by entry.S
 128  * to the lowcore fields user_timer, system_timer & steal_clock.
 129  */
 130 void vtime_account_irq_enter(struct task_struct *tsk)
 131 {
 132         struct thread_info *ti = task_thread_info(tsk);
 133         u64 timer, system;
 134
 135         WARN_ON_ONCE(!irqs_disabled());
 136
 137         timer = S390_lowcore.last_update_timer;
 138         S390_lowcore.last_update_timer = get_vtimer();
 139         S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
 140
 141         system = S390_lowcore.system_timer - ti->system_timer;
 142         S390_lowcore.steal_timer -= system;
 143         ti->system_timer = S390_lowcore.system_timer;
 144         account_system_time(tsk, 0, system, system);
 145
 146         virt_timer_forward(system);
 147 }
 148 EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
 149
 150 void vtime_account_system(struct task_struct *tsk)
 151 __attribute__((alias("vtime_account_irq_enter")));
 152 EXPORT_SYMBOL_GPL(vtime_account_system);
 153
 154 void __kprobes vtime_stop_cpu(void)
 155 {
 156         struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
 157         unsigned long long idle_time;
 158         unsigned long psw_mask;
 159
 160         trace_hardirqs_on();
 161
 162         /* Wait for external, I/O or machine check interrupt. */
 163         psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT |
 164                 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
 165         idle->nohz_delay = 0;
 166
 167         /* Call the assembler magic in entry.S */
 168         psw_idle(idle, psw_mask);
 169
 170         /* Account time spent with enabled wait psw loaded as idle time. */
 171         idle->sequence++;
 172         smp_wmb();
 173         idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
 174         idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
 175         idle->idle_time += idle_time;
 176         idle->idle_count++;
 177         account_idle_time(idle_time);
 178         smp_wmb();
 179         idle->sequence++;
 180 }
 181
 182 cputime64_t s390_get_idle_time(int cpu)
 183 {
 184         struct s390_idle_data *idle = &per_cpu(s390_idle, cpu);
 185         unsigned long long now, idle_enter, idle_exit;
 186         unsigned int sequence;
 187
 188         do {
 189                 now = get_tod_clock();
 190                 sequence = ACCESS_ONCE(idle->sequence);
 191                 idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
 192                 idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
 193         } while ((sequence & 1) || (idle->sequence != sequence));
 194         return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0;
 195 }
 196
 197 /*
 198  * Sorted add to a list. List is linear searched until first bigger
 199  * element is found.
 200  */
 201 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
 202 {
 203         struct vtimer_list *tmp;
 204
 205         list_for_each_entry(tmp, head, entry) {
 206                 if (tmp->expires > timer->expires) {
 207                         list_add_tail(&timer->entry, &tmp->entry);
 208                         return;
 209                 }
 210         }
 211         list_add_tail(&timer->entry, head);
 212 }
 213
 214 /*
 215  * Handler for expired virtual CPU timer.
 216  */
 217 static void virt_timer_expire(void)
 218 {
 219         struct vtimer_list *timer, *tmp;
 220         unsigned long elapsed;
 221         LIST_HEAD(cb_list);
 222
 223         /* walk timer list, fire all expired timers */
 224         spin_lock(&virt_timer_lock);
 225         elapsed = atomic64_read(&virt_timer_elapsed);
 226         list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
 227                 if (timer->expires < elapsed)
 228                         /* move expired timer to the callback queue */
 229                         list_move_tail(&timer->entry, &cb_list);
 230                 else
 231                         timer->expires -= elapsed;
 232         }
 233         if (!list_empty(&virt_timer_list)) {
 234                 timer = list_first_entry(&virt_timer_list,
 235                                          struct vtimer_list, entry);
 236                 atomic64_set(&virt_timer_current, timer->expires);
 237         }
 238         atomic64_sub(elapsed, &virt_timer_elapsed);
 239         spin_unlock(&virt_timer_lock);
 240
 241         /* Do callbacks and recharge periodic timers */
 242         list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
 243                 list_del_init(&timer->entry);
 244                 timer->function(timer->data);
 245                 if (timer->interval) {
 246                         /* Recharge interval timer */
 247                         timer->expires = timer->interval +
 248                                 atomic64_read(&virt_timer_elapsed);
 249                         spin_lock(&virt_timer_lock);
 250                         list_add_sorted(timer, &virt_timer_list);
 251                         spin_unlock(&virt_timer_lock);
 252                 }
 253         }
 254 }
 255
 256 void init_virt_timer(struct vtimer_list *timer)
 257 {
 258         timer->function = NULL;
 259         INIT_LIST_HEAD(&timer->entry);
 260 }
 261 EXPORT_SYMBOL(init_virt_timer);
 262
 263 static inline int vtimer_pending(struct vtimer_list *timer)
 264 {
 265         return !list_empty(&timer->entry);
 266 }
 267
 268 static void internal_add_vtimer(struct vtimer_list *timer)
 269 {
 270         if (list_empty(&virt_timer_list)) {
 271                 /* First timer, just program it. */
 272                 atomic64_set(&virt_timer_current, timer->expires);
 273                 atomic64_set(&virt_timer_elapsed, 0);
 274                 list_add(&timer->entry, &virt_timer_list);
 275         } else {
 276                 /* Update timer against current base. */
 277                 timer->expires += atomic64_read(&virt_timer_elapsed);
 278                 if (likely((s64) timer->expires <
 279                            (s64) atomic64_read(&virt_timer_current)))
 280                         /* The new timer expires before the current timer. */
 281                         atomic64_set(&virt_timer_current, timer->expires);
 282                 /* Insert new timer into the list. */
 283                 list_add_sorted(timer, &virt_timer_list);
 284         }
 285 }
 286
 287 static void __add_vtimer(struct vtimer_list *timer, int periodic)
 288 {
 289         unsigned long flags;
 290
 291         timer->interval = periodic ? timer->expires : 0;
 292         spin_lock_irqsave(&virt_timer_lock, flags);
 293         internal_add_vtimer(timer);
 294         spin_unlock_irqrestore(&virt_timer_lock, flags);
 295 }
 296
 297 /*
 298  * add_virt_timer - add an oneshot virtual CPU timer
 299  */
 300 void add_virt_timer(struct vtimer_list *timer)
 301 {
 302         __add_vtimer(timer, 0);
 303 }
 304 EXPORT_SYMBOL(add_virt_timer);
 305
 306 /*
 307  * add_virt_timer_int - add an interval virtual CPU timer
 308  */
 309 void add_virt_timer_periodic(struct vtimer_list *timer)
 310 {
 311         __add_vtimer(timer, 1);
 312 }
 313 EXPORT_SYMBOL(add_virt_timer_periodic);
 314
 315 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
 316 {
 317         unsigned long flags;
 318         int rc;
 319
 320         BUG_ON(!timer->function);
 321
 322         if (timer->expires == expires && vtimer_pending(timer))
 323                 return 1;
 324         spin_lock_irqsave(&virt_timer_lock, flags);
 325         rc = vtimer_pending(timer);
 326         if (rc)
 327                 list_del_init(&timer->entry);
 328         timer->interval = periodic ? expires : 0;
 329         timer->expires = expires;
 330         internal_add_vtimer(timer);
 331         spin_unlock_irqrestore(&virt_timer_lock, flags);
 332         return rc;
 333 }
 334
 335 /*
 336  * returns whether it has modified a pending timer (1) or not (0)
 337  */
 338 int mod_virt_timer(struct vtimer_list *timer, u64 expires)
 339 {
 340         return __mod_vtimer(timer, expires, 0);
 341 }
 342 EXPORT_SYMBOL(mod_virt_timer);
 343
 344 /*
 345  * returns whether it has modified a pending timer (1) or not (0)
 346  */
 347 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
 348 {
 349         return __mod_vtimer(timer, expires, 1);
 350 }
 351 EXPORT_SYMBOL(mod_virt_timer_periodic);
 352
 353 /*
 354  * Delete a virtual timer.
 355  *
 356  * returns whether the deleted timer was pending (1) or not (0)
 357  */
 358 int del_virt_timer(struct vtimer_list *timer)
 359 {
 360         unsigned long flags;
 361
 362         if (!vtimer_pending(timer))
 363                 return 0;
 364         spin_lock_irqsave(&virt_timer_lock, flags);
 365         list_del_init(&timer->entry);
 366         spin_unlock_irqrestore(&virt_timer_lock, flags);
 367         return 1;
 368 }
 369 EXPORT_SYMBOL(del_virt_timer);
 370
 371 /*
 372  * Start the virtual CPU timer on the current CPU.
 373  */
 374 void init_cpu_vtimer(void)
 375 {
 376         /* set initial cpu timer */
 377         set_vtimer(VTIMER_MAX_SLICE);
 378 }
 379
 380 static int s390_nohz_notify(struct notifier_block *self, unsigned long action,
 381                             void *hcpu)
 382 {
 383         struct s390_idle_data *idle;
 384         long cpu = (long) hcpu;
 385
 386         idle = &per_cpu(s390_idle, cpu);
 387         switch (action & ~CPU_TASKS_FROZEN) {
 388         case CPU_DYING:
 389                 idle->nohz_delay = 0;
 390         default:
 391                 break;
 392         }
 393         return NOTIFY_OK;
 394 }
 395
 396 void __init vtime_init(void)
 397 {
 398         /* Enable cpu timer interrupts on the boot cpu. */
 399         init_cpu_vtimer();
 400         cpu_notifier(s390_nohz_notify, 0);
 401 }