drivers/cpuidle/cpuidle.c

   1 /*
   2  * cpuidle.c - core cpuidle infrastructure
   3  *
   4  * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
   5  *               Shaohua Li <shaohua.li@intel.com>
   6  *               Adam Belay <abelay@novell.com>
   7  *
   8  * This code is licenced under the GPL.
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/mutex.h>
  13 #include <linux/sched.h>
  14 #include <linux/notifier.h>
  15 #include <linux/pm_qos_params.h>
  16 #include <linux/cpu.h>
  17 #include <linux/cpuidle.h>
  18 #include <linux/ktime.h>
  19 #include <linux/hrtimer.h>
  20 #include <trace/events/power.h>
  21
  22 #include "cpuidle.h"
  23
  24 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
  25
  26 DEFINE_MUTEX(cpuidle_lock);
  27 LIST_HEAD(cpuidle_detected_devices);
  28 static void (*pm_idle_old)(void);
  29
  30 static int enabled_devices;
  31 static int off __read_mostly;
  32
  33 int cpuidle_disabled(void)
  34 {
  35         return off;
  36 }
  37
  38 #if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT)
  39 static void cpuidle_kick_cpus(void)
  40 {
  41         cpu_idle_wait();
  42 }
  43 #elif defined(CONFIG_SMP)
  44 # error "Arch needs cpu_idle_wait() equivalent here"
  45 #else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */
  46 static void cpuidle_kick_cpus(void) {}
  47 #endif
  48
  49 static int __cpuidle_register_device(struct cpuidle_device *dev);
  50
  51 /**
  52  * cpuidle_idle_call - the main idle loop
  53  *
  54  * NOTE: no locks or semaphores should be used here
  55  */
  56 static void cpuidle_idle_call(void)
  57 {
  58         struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
  59         struct cpuidle_state *target_state;
  60         int next_state;
  61
  62         /* check if the device is ready */
  63         if (!dev || !dev->enabled) {
  64                 if (pm_idle_old)
  65                         pm_idle_old();
  66                 else
  67 #if defined(CONFIG_ARCH_HAS_DEFAULT_IDLE)
  68                         default_idle();
  69 #else
  70                         local_irq_enable();
  71 #endif
  72                 return;
  73         }
  74
  75 #if 0
  76         /* shows regressions, re-enable for 2.6.29 */
  77         /*
  78          * run any timers that can be run now, at this point
  79          * before calculating the idle duration etc.
  80          */
  81         hrtimer_peek_ahead_timers();
  82 #endif
  83
  84         /*
  85          * Call the device's prepare function before calling the
  86          * governor's select function.  ->prepare gives the device's
  87          * cpuidle driver a chance to update any dynamic information
  88          * of its cpuidle states for the current idle period, e.g.
  89          * state availability, latencies, residencies, etc.
  90          */
  91         if (dev->prepare)
  92                 dev->prepare(dev);
  93
  94         /* ask the governor for the next state */
  95         next_state = cpuidle_curr_governor->select(dev);
  96         if (need_resched()) {
  97                 local_irq_enable();
  98                 return;
  99         }
 100
 101         target_state = &dev->states[next_state];
 102
 103         /* enter the state and update stats */
 104         dev->last_state = target_state;
 105
 106         trace_power_start(POWER_CSTATE, next_state, dev->cpu);
 107         trace_cpu_idle(next_state, dev->cpu);
 108
 109         dev->last_residency = target_state->enter(dev, target_state);
 110
 111         trace_power_end(dev->cpu);
 112         trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);
 113
 114         if (dev->last_state)
 115                 target_state = dev->last_state;
 116
 117         target_state->time += (unsigned long long)dev->last_residency;
 118         target_state->usage++;
 119
 120         /* give the governor an opportunity to reflect on the outcome */
 121         if (cpuidle_curr_governor->reflect)
 122                 cpuidle_curr_governor->reflect(dev);
 123 }
 124
 125 /**
 126  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
 127  */
 128 void cpuidle_install_idle_handler(void)
 129 {
 130         if (enabled_devices && (pm_idle != cpuidle_idle_call)) {
 131                 /* Make sure all changes finished before we switch to new idle */
 132                 smp_wmb();
 133                 pm_idle = cpuidle_idle_call;
 134         }
 135 }
 136
 137 /**
 138  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
 139  */
 140 void cpuidle_uninstall_idle_handler(void)
 141 {
 142         if (enabled_devices && pm_idle_old && (pm_idle != pm_idle_old)) {
 143                 pm_idle = pm_idle_old;
 144                 cpuidle_kick_cpus();
 145         }
 146 }
 147
 148 /**
 149  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
 150  */
 151 void cpuidle_pause_and_lock(void)
 152 {
 153         mutex_lock(&cpuidle_lock);
 154         cpuidle_uninstall_idle_handler();
 155 }
 156
 157 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
 158
 159 /**
 160  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
 161  */
 162 void cpuidle_resume_and_unlock(void)
 163 {
 164         cpuidle_install_idle_handler();
 165         mutex_unlock(&cpuidle_lock);
 166 }
 167
 168 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
 169
 170 #ifdef CONFIG_ARCH_HAS_CPU_RELAX
 171 static int poll_idle(struct cpuidle_device *dev, struct cpuidle_state *st)
 172 {
 173         ktime_t t1, t2;
 174         s64 diff;
 175         int ret;
 176
 177         t1 = ktime_get();
 178         local_irq_enable();
 179         while (!need_resched())
 180                 cpu_relax();
 181
 182         t2 = ktime_get();
 183         diff = ktime_to_us(ktime_sub(t2, t1));
 184         if (diff > INT_MAX)
 185                 diff = INT_MAX;
 186
 187         ret = (int) diff;
 188         return ret;
 189 }
 190
 191 static void poll_idle_init(struct cpuidle_device *dev)
 192 {
 193         struct cpuidle_state *state = &dev->states[0];
 194
 195         cpuidle_set_statedata(state, NULL);
 196
 197         snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
 198         snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
 199         state->exit_latency = 0;
 200         state->target_residency = 0;
 201         state->power_usage = -1;
 202         state->flags = 0;
 203         state->enter = poll_idle;
 204 }
 205 #else
 206 static void poll_idle_init(struct cpuidle_device *dev) {}
 207 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */
 208
 209 /**
 210  * cpuidle_enable_device - enables idle PM for a CPU
 211  * @dev: the CPU
 212  *
 213  * This function must be called between cpuidle_pause_and_lock and
 214  * cpuidle_resume_and_unlock when used externally.
 215  */
 216 int cpuidle_enable_device(struct cpuidle_device *dev)
 217 {
 218         int ret, i;
 219
 220         if (dev->enabled)
 221                 return 0;
 222         if (!cpuidle_get_driver() || !cpuidle_curr_governor)
 223                 return -EIO;
 224         if (!dev->state_count)
 225                 return -EINVAL;
 226
 227         if (dev->registered == 0) {
 228                 ret = __cpuidle_register_device(dev);
 229                 if (ret)
 230                         return ret;
 231         }
 232
 233         poll_idle_init(dev);
 234
 235         if ((ret = cpuidle_add_state_sysfs(dev)))
 236                 return ret;
 237
 238         if (cpuidle_curr_governor->enable &&
 239             (ret = cpuidle_curr_governor->enable(dev)))
 240                 goto fail_sysfs;
 241
 242         for (i = 0; i < dev->state_count; i++) {
 243                 dev->states[i].usage = 0;
 244                 dev->states[i].time = 0;
 245         }
 246         dev->last_residency = 0;
 247         dev->last_state = NULL;
 248
 249         smp_wmb();
 250
 251         dev->enabled = 1;
 252
 253         enabled_devices++;
 254         return 0;
 255
 256 fail_sysfs:
 257         cpuidle_remove_state_sysfs(dev);
 258
 259         return ret;
 260 }
 261
 262 EXPORT_SYMBOL_GPL(cpuidle_enable_device);
 263
 264 /**
 265  * cpuidle_disable_device - disables idle PM for a CPU
 266  * @dev: the CPU
 267  *
 268  * This function must be called between cpuidle_pause_and_lock and
 269  * cpuidle_resume_and_unlock when used externally.
 270  */
 271 void cpuidle_disable_device(struct cpuidle_device *dev)
 272 {
 273         if (!dev->enabled)
 274                 return;
 275         if (!cpuidle_get_driver() || !cpuidle_curr_governor)
 276                 return;
 277
 278         dev->enabled = 0;
 279
 280         if (cpuidle_curr_governor->disable)
 281                 cpuidle_curr_governor->disable(dev);
 282
 283         cpuidle_remove_state_sysfs(dev);
 284         enabled_devices--;
 285 }
 286
 287 EXPORT_SYMBOL_GPL(cpuidle_disable_device);
 288
 289 /**
 290  * __cpuidle_register_device - internal register function called before register
 291  * and enable routines
 292  * @dev: the cpu
 293  *
 294  * cpuidle_lock mutex must be held before this is called
 295  */
 296 static int __cpuidle_register_device(struct cpuidle_device *dev)
 297 {
 298         int ret;
 299         struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
 300         struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
 301
 302         if (!sys_dev)
 303                 return -EINVAL;
 304         if (!try_module_get(cpuidle_driver->owner))
 305                 return -EINVAL;
 306
 307         init_completion(&dev->kobj_unregister);
 308
 309         /*
 310          * cpuidle driver should set the dev->power_specified bit
 311          * before registering the device if the driver provides
 312          * power_usage numbers.
 313          *
 314          * For those devices whose ->power_specified is not set,
 315          * we fill in power_usage with decreasing values as the
 316          * cpuidle code has an implicit assumption that state Cn
 317          * uses less power than C(n-1).
 318          *
 319          * With CONFIG_ARCH_HAS_CPU_RELAX, C0 is already assigned
 320          * an power value of -1.  So we use -2, -3, etc, for other
 321          * c-states.
 322          */
 323         if (!dev->power_specified) {
 324                 int i;
 325                 for (i = CPUIDLE_DRIVER_STATE_START; i < dev->state_count; i++)
 326                         dev->states[i].power_usage = -1 - i;
 327         }
 328
 329         per_cpu(cpuidle_devices, dev->cpu) = dev;
 330         list_add(&dev->device_list, &cpuidle_detected_devices);
 331         if ((ret = cpuidle_add_sysfs(sys_dev))) {
 332                 module_put(cpuidle_driver->owner);
 333                 return ret;
 334         }
 335
 336         dev->registered = 1;
 337         return 0;
 338 }
 339
 340 /**
 341  * cpuidle_register_device - registers a CPU's idle PM feature
 342  * @dev: the cpu
 343  */
 344 int cpuidle_register_device(struct cpuidle_device *dev)
 345 {
 346         int ret;
 347
 348         mutex_lock(&cpuidle_lock);
 349
 350         if ((ret = __cpuidle_register_device(dev))) {
 351                 mutex_unlock(&cpuidle_lock);
 352                 return ret;
 353         }
 354
 355         cpuidle_enable_device(dev);
 356         cpuidle_install_idle_handler();
 357
 358         mutex_unlock(&cpuidle_lock);
 359
 360         return 0;
 361
 362 }
 363
 364 EXPORT_SYMBOL_GPL(cpuidle_register_device);
 365
 366 /**
 367  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
 368  * @dev: the cpu
 369  */
 370 void cpuidle_unregister_device(struct cpuidle_device *dev)
 371 {
 372         struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
 373         struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
 374
 375         if (dev->registered == 0)
 376                 return;
 377
 378         cpuidle_pause_and_lock();
 379
 380         cpuidle_disable_device(dev);
 381
 382         cpuidle_remove_sysfs(sys_dev);
 383         list_del(&dev->device_list);
 384         wait_for_completion(&dev->kobj_unregister);
 385         per_cpu(cpuidle_devices, dev->cpu) = NULL;
 386
 387         cpuidle_resume_and_unlock();
 388
 389         module_put(cpuidle_driver->owner);
 390 }
 391
 392 EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
 393
 394 #ifdef CONFIG_SMP
 395
 396 static void smp_callback(void *v)
 397 {
 398         /* we already woke the CPU up, nothing more to do */
 399 }
 400
 401 /*
 402  * This function gets called when a part of the kernel has a new latency
 403  * requirement.  This means we need to get all processors out of their C-state,
 404  * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
 405  * wakes them all right up.
 406  */
 407 static int cpuidle_latency_notify(struct notifier_block *b,
 408                 unsigned long l, void *v)
 409 {
 410         smp_call_function(smp_callback, NULL, 1);
 411         return NOTIFY_OK;
 412 }
 413
 414 static struct notifier_block cpuidle_latency_notifier = {
 415         .notifier_call = cpuidle_latency_notify,
 416 };
 417
 418 static inline void latency_notifier_init(struct notifier_block *n)
 419 {
 420         pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
 421 }
 422
 423 #else /* CONFIG_SMP */
 424
 425 #define latency_notifier_init(x) do { } while (0)
 426
 427 #endif /* CONFIG_SMP */
 428
 429 /**
 430  * cpuidle_init - core initializer
 431  */
 432 static int __init cpuidle_init(void)
 433 {
 434         int ret;
 435
 436         if (cpuidle_disabled())
 437                 return -ENODEV;
 438
 439         pm_idle_old = pm_idle;
 440
 441         ret = cpuidle_add_class_sysfs(&cpu_sysdev_class);
 442         if (ret)
 443                 return ret;
 444
 445         latency_notifier_init(&cpuidle_latency_notifier);
 446
 447         return 0;
 448 }
 449
 450 module_param(off, int, 0444);
 451 core_initcall(cpuidle_init);