| 1 | /* |
| 2 | * ladder.c - the residency ladder algorithm |
| 3 | * |
| 4 | * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com> |
| 5 | * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com> |
| 6 | * Copyright (C) 2004, 2005 Dominik Brodowski <linux@brodo.de> |
| 7 | * |
| 8 | * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> |
| 9 | * Shaohua Li <shaohua.li@intel.com> |
| 10 | * Adam Belay <abelay@novell.com> |
| 11 | * |
| 12 | * This code is licenced under the GPL. |
| 13 | */ |
| 14 | |
| 15 | #include <linux/kernel.h> |
| 16 | #include <linux/cpuidle.h> |
| 17 | #include <linux/pm_qos.h> |
| 18 | #include <linux/module.h> |
| 19 | #include <linux/jiffies.h> |
| 20 | #include <linux/tick.h> |
| 21 | |
| 22 | #include <asm/io.h> |
| 23 | #include <asm/uaccess.h> |
| 24 | |
| 25 | #define PROMOTION_COUNT 4 |
| 26 | #define DEMOTION_COUNT 1 |
| 27 | |
| 28 | struct ladder_device_state { |
| 29 | struct { |
| 30 | u32 promotion_count; |
| 31 | u32 demotion_count; |
| 32 | u32 promotion_time; |
| 33 | u32 demotion_time; |
| 34 | } threshold; |
| 35 | struct { |
| 36 | int promotion_count; |
| 37 | int demotion_count; |
| 38 | } stats; |
| 39 | }; |
| 40 | |
| 41 | struct ladder_device { |
| 42 | struct ladder_device_state states[CPUIDLE_STATE_MAX]; |
| 43 | int last_state_idx; |
| 44 | }; |
| 45 | |
| 46 | static DEFINE_PER_CPU(struct ladder_device, ladder_devices); |
| 47 | |
| 48 | /** |
| 49 | * ladder_do_selection - prepares private data for a state change |
| 50 | * @ldev: the ladder device |
| 51 | * @old_idx: the current state index |
| 52 | * @new_idx: the new target state index |
| 53 | */ |
| 54 | static inline void ladder_do_selection(struct ladder_device *ldev, |
| 55 | int old_idx, int new_idx) |
| 56 | { |
| 57 | ldev->states[old_idx].stats.promotion_count = 0; |
| 58 | ldev->states[old_idx].stats.demotion_count = 0; |
| 59 | ldev->last_state_idx = new_idx; |
| 60 | } |
| 61 | |
| 62 | /** |
| 63 | * ladder_select_state - selects the next state to enter |
| 64 | * @drv: cpuidle driver |
| 65 | * @dev: the CPU |
| 66 | */ |
| 67 | static int ladder_select_state(struct cpuidle_driver *drv, |
| 68 | struct cpuidle_device *dev) |
| 69 | { |
| 70 | struct ladder_device *ldev = this_cpu_ptr(&ladder_devices); |
| 71 | struct ladder_device_state *last_state; |
| 72 | int last_residency, last_idx = ldev->last_state_idx; |
| 73 | int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY); |
| 74 | |
| 75 | /* Special case when user has set very strict latency requirement */ |
| 76 | if (unlikely(latency_req == 0)) { |
| 77 | ladder_do_selection(ldev, last_idx, 0); |
| 78 | return 0; |
| 79 | } |
| 80 | |
| 81 | last_state = &ldev->states[last_idx]; |
| 82 | |
| 83 | last_residency = cpuidle_get_last_residency(dev) - drv->states[last_idx].exit_latency; |
| 84 | |
| 85 | /* consider promotion */ |
| 86 | if (last_idx < drv->state_count - 1 && |
| 87 | !drv->states[last_idx + 1].disabled && |
| 88 | !dev->states_usage[last_idx + 1].disable && |
| 89 | last_residency > last_state->threshold.promotion_time && |
| 90 | drv->states[last_idx + 1].exit_latency <= latency_req) { |
| 91 | last_state->stats.promotion_count++; |
| 92 | last_state->stats.demotion_count = 0; |
| 93 | if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) { |
| 94 | ladder_do_selection(ldev, last_idx, last_idx + 1); |
| 95 | return last_idx + 1; |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | /* consider demotion */ |
| 100 | if (last_idx > CPUIDLE_DRIVER_STATE_START && |
| 101 | (drv->states[last_idx].disabled || |
| 102 | dev->states_usage[last_idx].disable || |
| 103 | drv->states[last_idx].exit_latency > latency_req)) { |
| 104 | int i; |
| 105 | |
| 106 | for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) { |
| 107 | if (drv->states[i].exit_latency <= latency_req) |
| 108 | break; |
| 109 | } |
| 110 | ladder_do_selection(ldev, last_idx, i); |
| 111 | return i; |
| 112 | } |
| 113 | |
| 114 | if (last_idx > CPUIDLE_DRIVER_STATE_START && |
| 115 | last_residency < last_state->threshold.demotion_time) { |
| 116 | last_state->stats.demotion_count++; |
| 117 | last_state->stats.promotion_count = 0; |
| 118 | if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) { |
| 119 | ladder_do_selection(ldev, last_idx, last_idx - 1); |
| 120 | return last_idx - 1; |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | /* otherwise remain at the current state */ |
| 125 | return last_idx; |
| 126 | } |
| 127 | |
| 128 | /** |
| 129 | * ladder_enable_device - setup for the governor |
| 130 | * @drv: cpuidle driver |
| 131 | * @dev: the CPU |
| 132 | */ |
| 133 | static int ladder_enable_device(struct cpuidle_driver *drv, |
| 134 | struct cpuidle_device *dev) |
| 135 | { |
| 136 | int i; |
| 137 | struct ladder_device *ldev = &per_cpu(ladder_devices, dev->cpu); |
| 138 | struct ladder_device_state *lstate; |
| 139 | struct cpuidle_state *state; |
| 140 | |
| 141 | ldev->last_state_idx = CPUIDLE_DRIVER_STATE_START; |
| 142 | |
| 143 | for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) { |
| 144 | state = &drv->states[i]; |
| 145 | lstate = &ldev->states[i]; |
| 146 | |
| 147 | lstate->stats.promotion_count = 0; |
| 148 | lstate->stats.demotion_count = 0; |
| 149 | |
| 150 | lstate->threshold.promotion_count = PROMOTION_COUNT; |
| 151 | lstate->threshold.demotion_count = DEMOTION_COUNT; |
| 152 | |
| 153 | if (i < drv->state_count - 1) |
| 154 | lstate->threshold.promotion_time = state->exit_latency; |
| 155 | if (i > CPUIDLE_DRIVER_STATE_START) |
| 156 | lstate->threshold.demotion_time = state->exit_latency; |
| 157 | } |
| 158 | |
| 159 | return 0; |
| 160 | } |
| 161 | |
| 162 | /** |
| 163 | * ladder_reflect - update the correct last_state_idx |
| 164 | * @dev: the CPU |
| 165 | * @index: the index of actual state entered |
| 166 | */ |
| 167 | static void ladder_reflect(struct cpuidle_device *dev, int index) |
| 168 | { |
| 169 | struct ladder_device *ldev = this_cpu_ptr(&ladder_devices); |
| 170 | if (index > 0) |
| 171 | ldev->last_state_idx = index; |
| 172 | } |
| 173 | |
| 174 | static struct cpuidle_governor ladder_governor = { |
| 175 | .name = "ladder", |
| 176 | .rating = 10, |
| 177 | .enable = ladder_enable_device, |
| 178 | .select = ladder_select_state, |
| 179 | .reflect = ladder_reflect, |
| 180 | .owner = THIS_MODULE, |
| 181 | }; |
| 182 | |
| 183 | /** |
| 184 | * init_ladder - initializes the governor |
| 185 | */ |
| 186 | static int __init init_ladder(void) |
| 187 | { |
| 188 | /* |
| 189 | * When NO_HZ is disabled, or when booting with nohz=off, the ladder |
| 190 | * governor is better so give it a higher rating than the menu |
| 191 | * governor. |
| 192 | */ |
| 193 | if (!tick_nohz_enabled) |
| 194 | ladder_governor.rating = 25; |
| 195 | |
| 196 | return cpuidle_register_governor(&ladder_governor); |
| 197 | } |
| 198 | |
| 199 | postcore_initcall(init_ladder); |