drivers/cpuidle/cpuidle.c

   1 /*
   2  * cpuidle.c - core cpuidle infrastructure
   3  *
   4  * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
   5  *               Shaohua Li <shaohua.li@intel.com>
   6  *               Adam Belay <abelay@novell.com>
   7  *
   8  * This code is licenced under the GPL.
   9  */
  10
  11 #include <linux/clockchips.h>
  12 #include <linux/kernel.h>
  13 #include <linux/mutex.h>
  14 #include <linux/sched.h>
  15 #include <linux/notifier.h>
  16 #include <linux/pm_qos.h>
  17 #include <linux/cpu.h>
  18 #include <linux/cpuidle.h>
  19 #include <linux/ktime.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/module.h>
  22 #include <trace/events/power.h>
  23
  24 #include "cpuidle.h"
  25
  26 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
  27
  28 DEFINE_MUTEX(cpuidle_lock);
  29 LIST_HEAD(cpuidle_detected_devices);
  30
  31 static int enabled_devices;
  32 static int off __read_mostly;
  33 static int initialized __read_mostly;
  34
  35 int cpuidle_disabled(void)
  36 {
  37         return off;
  38 }
  39 void disable_cpuidle(void)
  40 {
  41         off = 1;
  42 }
  43
  44 static int __cpuidle_register_device(struct cpuidle_device *dev);
  45
  46 static inline int cpuidle_enter(struct cpuidle_device *dev,
  47                                 struct cpuidle_driver *drv, int index)
  48 {
  49         struct cpuidle_state *target_state = &drv->states[index];
  50         return target_state->enter(dev, drv, index);
  51 }
  52
  53 static inline int cpuidle_enter_tk(struct cpuidle_device *dev,
  54                                struct cpuidle_driver *drv, int index)
  55 {
  56         return cpuidle_wrap_enter(dev, drv, index, cpuidle_enter);
  57 }
  58
  59 typedef int (*cpuidle_enter_t)(struct cpuidle_device *dev,
  60                                struct cpuidle_driver *drv, int index);
  61
  62 static cpuidle_enter_t cpuidle_enter_ops;
  63
  64 /**
  65  * cpuidle_play_dead - cpu off-lining
  66  *
  67  * Returns in case of an error or no driver
  68  */
  69 int cpuidle_play_dead(void)
  70 {
  71         struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
  72         struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
  73         int i;
  74
  75         if (!drv)
  76                 return -ENODEV;
  77
  78         /* Find lowest-power state that supports long-term idle */
  79         for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--)
  80                 if (drv->states[i].enter_dead)
  81                         return drv->states[i].enter_dead(dev, i);
  82
  83         return -ENODEV;
  84 }
  85
  86 /**
  87  * cpuidle_enter_state - enter the state and update stats
  88  * @dev: cpuidle device for this cpu
  89  * @drv: cpuidle driver for this cpu
  90  * @next_state: index into drv->states of the state to enter
  91  */
  92 int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
  93                 int next_state)
  94 {
  95         int entered_state;
  96
  97         entered_state = cpuidle_enter_ops(dev, drv, next_state);
  98
  99         if (entered_state >= 0) {
 100                 /* Update cpuidle counters */
 101                 /* This can be moved to within driver enter routine
 102                  * but that results in multiple copies of same code.
 103                  */
 104                 dev->states_usage[entered_state].time += dev->last_residency;
 105                 dev->states_usage[entered_state].usage++;
 106         } else {
 107                 dev->last_residency = 0;
 108         }
 109
 110         return entered_state;
 111 }
 112
 113 /**
 114  * cpuidle_idle_call - the main idle loop
 115  *
 116  * NOTE: no locks or semaphores should be used here
 117  * return non-zero on failure
 118  */
 119 int cpuidle_idle_call(void)
 120 {
 121         struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
 122         struct cpuidle_driver *drv;
 123         int next_state, entered_state;
 124
 125         if (off)
 126                 return -ENODEV;
 127
 128         if (!initialized)
 129                 return -ENODEV;
 130
 131         /* check if the device is ready */
 132         if (!dev || !dev->enabled)
 133                 return -EBUSY;
 134
 135         drv = cpuidle_get_cpu_driver(dev);
 136
 137         /* ask the governor for the next state */
 138         next_state = cpuidle_curr_governor->select(drv, dev);
 139         if (need_resched()) {
 140                 dev->last_residency = 0;
 141                 /* give the governor an opportunity to reflect on the outcome */
 142                 if (cpuidle_curr_governor->reflect)
 143                         cpuidle_curr_governor->reflect(dev, next_state);
 144                 local_irq_enable();
 145                 return 0;
 146         }
 147
 148         trace_cpu_idle_rcuidle(next_state, dev->cpu);
 149
 150         if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP)
 151                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
 152                                    &dev->cpu);
 153
 154         if (cpuidle_state_is_coupled(dev, drv, next_state))
 155                 entered_state = cpuidle_enter_state_coupled(dev, drv,
 156                                                             next_state);
 157         else
 158                 entered_state = cpuidle_enter_state(dev, drv, next_state);
 159
 160         if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP)
 161                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
 162                                    &dev->cpu);
 163
 164         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
 165
 166         /* give the governor an opportunity to reflect on the outcome */
 167         if (cpuidle_curr_governor->reflect)
 168                 cpuidle_curr_governor->reflect(dev, entered_state);
 169
 170         return 0;
 171 }
 172
 173 /**
 174  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
 175  */
 176 void cpuidle_install_idle_handler(void)
 177 {
 178         if (enabled_devices) {
 179                 /* Make sure all changes finished before we switch to new idle */
 180                 smp_wmb();
 181                 initialized = 1;
 182         }
 183 }
 184
 185 /**
 186  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
 187  */
 188 void cpuidle_uninstall_idle_handler(void)
 189 {
 190         if (enabled_devices) {
 191                 initialized = 0;
 192                 kick_all_cpus_sync();
 193         }
 194 }
 195
 196 /**
 197  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
 198  */
 199 void cpuidle_pause_and_lock(void)
 200 {
 201         mutex_lock(&cpuidle_lock);
 202         cpuidle_uninstall_idle_handler();
 203 }
 204
 205 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
 206
 207 /**
 208  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
 209  */
 210 void cpuidle_resume_and_unlock(void)
 211 {
 212         cpuidle_install_idle_handler();
 213         mutex_unlock(&cpuidle_lock);
 214 }
 215
 216 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
 217
 218 /* Currently used in suspend/resume path to suspend cpuidle */
 219 void cpuidle_pause(void)
 220 {
 221         mutex_lock(&cpuidle_lock);
 222         cpuidle_uninstall_idle_handler();
 223         mutex_unlock(&cpuidle_lock);
 224 }
 225
 226 /* Currently used in suspend/resume path to resume cpuidle */
 227 void cpuidle_resume(void)
 228 {
 229         mutex_lock(&cpuidle_lock);
 230         cpuidle_install_idle_handler();
 231         mutex_unlock(&cpuidle_lock);
 232 }
 233
 234 /**
 235  * cpuidle_wrap_enter - performs timekeeping and irqen around enter function
 236  * @dev: pointer to a valid cpuidle_device object
 237  * @drv: pointer to a valid cpuidle_driver object
 238  * @index: index of the target cpuidle state.
 239  */
 240 int cpuidle_wrap_enter(struct cpuidle_device *dev,
 241                                 struct cpuidle_driver *drv, int index,
 242                                 int (*enter)(struct cpuidle_device *dev,
 243                                         struct cpuidle_driver *drv, int index))
 244 {
 245         ktime_t time_start, time_end;
 246         s64 diff;
 247
 248         time_start = ktime_get();
 249
 250         index = enter(dev, drv, index);
 251
 252         time_end = ktime_get();
 253
 254         local_irq_enable();
 255
 256         diff = ktime_to_us(ktime_sub(time_end, time_start));
 257         if (diff > INT_MAX)
 258                 diff = INT_MAX;
 259
 260         dev->last_residency = (int) diff;
 261
 262         return index;
 263 }
 264
 265 #ifdef CONFIG_ARCH_HAS_CPU_RELAX
 266 static int poll_idle(struct cpuidle_device *dev,
 267                 struct cpuidle_driver *drv, int index)
 268 {
 269         ktime_t t1, t2;
 270         s64 diff;
 271
 272         t1 = ktime_get();
 273         local_irq_enable();
 274         while (!need_resched())
 275                 cpu_relax();
 276
 277         t2 = ktime_get();
 278         diff = ktime_to_us(ktime_sub(t2, t1));
 279         if (diff > INT_MAX)
 280                 diff = INT_MAX;
 281
 282         dev->last_residency = (int) diff;
 283
 284         return index;
 285 }
 286
 287 static void poll_idle_init(struct cpuidle_driver *drv)
 288 {
 289         struct cpuidle_state *state = &drv->states[0];
 290
 291         snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
 292         snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
 293         state->exit_latency = 0;
 294         state->target_residency = 0;
 295         state->power_usage = -1;
 296         state->flags = 0;
 297         state->enter = poll_idle;
 298         state->disabled = false;
 299 }
 300 #else
 301 static void poll_idle_init(struct cpuidle_driver *drv) {}
 302 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */
 303
 304 /**
 305  * cpuidle_enable_device - enables idle PM for a CPU
 306  * @dev: the CPU
 307  *
 308  * This function must be called between cpuidle_pause_and_lock and
 309  * cpuidle_resume_and_unlock when used externally.
 310  */
 311 int cpuidle_enable_device(struct cpuidle_device *dev)
 312 {
 313         int ret, i;
 314         struct cpuidle_driver *drv;
 315
 316         if (!dev)
 317                 return -EINVAL;
 318
 319         if (dev->enabled)
 320                 return 0;
 321
 322         drv = cpuidle_get_cpu_driver(dev);
 323
 324         if (!drv || !cpuidle_curr_governor)
 325                 return -EIO;
 326
 327         if (!dev->state_count)
 328                 dev->state_count = drv->state_count;
 329
 330         if (dev->registered == 0) {
 331                 ret = __cpuidle_register_device(dev);
 332                 if (ret)
 333                         return ret;
 334         }
 335
 336         cpuidle_enter_ops = drv->en_core_tk_irqen ?
 337                 cpuidle_enter_tk : cpuidle_enter;
 338
 339         poll_idle_init(drv);
 340
 341         ret = cpuidle_add_device_sysfs(dev);
 342         if (ret)
 343                 return ret;
 344
 345         if (cpuidle_curr_governor->enable &&
 346             (ret = cpuidle_curr_governor->enable(drv, dev)))
 347                 goto fail_sysfs;
 348
 349         for (i = 0; i < dev->state_count; i++) {
 350                 dev->states_usage[i].usage = 0;
 351                 dev->states_usage[i].time = 0;
 352         }
 353         dev->last_residency = 0;
 354
 355         smp_wmb();
 356
 357         dev->enabled = 1;
 358
 359         enabled_devices++;
 360         return 0;
 361
 362 fail_sysfs:
 363         cpuidle_remove_device_sysfs(dev);
 364
 365         return ret;
 366 }
 367
 368 EXPORT_SYMBOL_GPL(cpuidle_enable_device);
 369
 370 /**
 371  * cpuidle_disable_device - disables idle PM for a CPU
 372  * @dev: the CPU
 373  *
 374  * This function must be called between cpuidle_pause_and_lock and
 375  * cpuidle_resume_and_unlock when used externally.
 376  */
 377 void cpuidle_disable_device(struct cpuidle_device *dev)
 378 {
 379         struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
 380
 381         if (!dev || !dev->enabled)
 382                 return;
 383
 384         if (!drv || !cpuidle_curr_governor)
 385                 return;
 386
 387         dev->enabled = 0;
 388
 389         if (cpuidle_curr_governor->disable)
 390                 cpuidle_curr_governor->disable(drv, dev);
 391
 392         cpuidle_remove_device_sysfs(dev);
 393         enabled_devices--;
 394 }
 395
 396 EXPORT_SYMBOL_GPL(cpuidle_disable_device);
 397
 398 /**
 399  * __cpuidle_register_device - internal register function called before register
 400  * and enable routines
 401  * @dev: the cpu
 402  *
 403  * cpuidle_lock mutex must be held before this is called
 404  */
 405 static int __cpuidle_register_device(struct cpuidle_device *dev)
 406 {
 407         int ret;
 408         struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
 409
 410         if (!try_module_get(drv->owner))
 411                 return -EINVAL;
 412
 413         per_cpu(cpuidle_devices, dev->cpu) = dev;
 414         list_add(&dev->device_list, &cpuidle_detected_devices);
 415         ret = cpuidle_add_sysfs(dev);
 416         if (ret)
 417                 goto err_sysfs;
 418
 419         ret = cpuidle_coupled_register_device(dev);
 420         if (ret)
 421                 goto err_coupled;
 422
 423         dev->registered = 1;
 424         return 0;
 425
 426 err_coupled:
 427         cpuidle_remove_sysfs(dev);
 428 err_sysfs:
 429         list_del(&dev->device_list);
 430         per_cpu(cpuidle_devices, dev->cpu) = NULL;
 431         module_put(drv->owner);
 432         return ret;
 433 }
 434
 435 /**
 436  * cpuidle_register_device - registers a CPU's idle PM feature
 437  * @dev: the cpu
 438  */
 439 int cpuidle_register_device(struct cpuidle_device *dev)
 440 {
 441         int ret;
 442
 443         if (!dev)
 444                 return -EINVAL;
 445
 446         mutex_lock(&cpuidle_lock);
 447
 448         if ((ret = __cpuidle_register_device(dev))) {
 449                 mutex_unlock(&cpuidle_lock);
 450                 return ret;
 451         }
 452
 453         cpuidle_enable_device(dev);
 454         cpuidle_install_idle_handler();
 455
 456         mutex_unlock(&cpuidle_lock);
 457
 458         return 0;
 459
 460 }
 461
 462 EXPORT_SYMBOL_GPL(cpuidle_register_device);
 463
 464 /**
 465  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
 466  * @dev: the cpu
 467  */
 468 void cpuidle_unregister_device(struct cpuidle_device *dev)
 469 {
 470         struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
 471
 472         if (dev->registered == 0)
 473                 return;
 474
 475         cpuidle_pause_and_lock();
 476
 477         cpuidle_disable_device(dev);
 478
 479         cpuidle_remove_sysfs(dev);
 480         list_del(&dev->device_list);
 481         per_cpu(cpuidle_devices, dev->cpu) = NULL;
 482
 483         cpuidle_coupled_unregister_device(dev);
 484
 485         cpuidle_resume_and_unlock();
 486
 487         module_put(drv->owner);
 488 }
 489
 490 EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
 491
 492 #ifdef CONFIG_SMP
 493
 494 static void smp_callback(void *v)
 495 {
 496         /* we already woke the CPU up, nothing more to do */
 497 }
 498
 499 /*
 500  * This function gets called when a part of the kernel has a new latency
 501  * requirement.  This means we need to get all processors out of their C-state,
 502  * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
 503  * wakes them all right up.
 504  */
 505 static int cpuidle_latency_notify(struct notifier_block *b,
 506                 unsigned long l, void *v)
 507 {
 508         smp_call_function(smp_callback, NULL, 1);
 509         return NOTIFY_OK;
 510 }
 511
 512 static struct notifier_block cpuidle_latency_notifier = {
 513         .notifier_call = cpuidle_latency_notify,
 514 };
 515
 516 static inline void latency_notifier_init(struct notifier_block *n)
 517 {
 518         pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
 519 }
 520
 521 #else /* CONFIG_SMP */
 522
 523 #define latency_notifier_init(x) do { } while (0)
 524
 525 #endif /* CONFIG_SMP */
 526
 527 /**
 528  * cpuidle_init - core initializer
 529  */
 530 static int __init cpuidle_init(void)
 531 {
 532         int ret;
 533
 534         if (cpuidle_disabled())
 535                 return -ENODEV;
 536
 537         ret = cpuidle_add_interface(cpu_subsys.dev_root);
 538         if (ret)
 539                 return ret;
 540
 541         latency_notifier_init(&cpuidle_latency_notifier);
 542
 543         return 0;
 544 }
 545
 546 module_param(off, int, 0444);
 547 core_initcall(cpuidle_init);