drivers/cpufreq/cpufreq_governor.c

   1 /*
   2  * drivers/cpufreq/cpufreq_governor.c
   3  *
   4  * CPUFREQ governors common code
   5  *
   6  * Copyright    (C) 2001 Russell King
   7  *              (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
   8  *              (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
   9  *              (C) 2009 Alexander Clouter <alex@digriz.org.uk>
  10  *              (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/export.h>
  20 #include <linux/kernel_stat.h>
  21 #include <linux/slab.h>
  22
  23 #include "cpufreq_governor.h"
  24
  25 static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data)
  26 {
  27         if (have_governor_per_policy())
  28                 return dbs_data->cdata->attr_group_gov_pol;
  29         else
  30                 return dbs_data->cdata->attr_group_gov_sys;
  31 }
  32
  33 void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
  34 {
  35         struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
  36         struct od_dbs_tuners *od_tuners = dbs_data->tuners;
  37         struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
  38         struct cpufreq_policy *policy = cdbs->shared->policy;
  39         unsigned int sampling_rate;
  40         unsigned int max_load = 0;
  41         unsigned int ignore_nice;
  42         unsigned int j;
  43
  44         if (dbs_data->cdata->governor == GOV_ONDEMAND) {
  45                 struct od_cpu_dbs_info_s *od_dbs_info =
  46                                 dbs_data->cdata->get_cpu_dbs_info_s(cpu);
  47
  48                 /*
  49                  * Sometimes, the ondemand governor uses an additional
  50                  * multiplier to give long delays. So apply this multiplier to
  51                  * the 'sampling_rate', so as to keep the wake-up-from-idle
  52                  * detection logic a bit conservative.
  53                  */
  54                 sampling_rate = od_tuners->sampling_rate;
  55                 sampling_rate *= od_dbs_info->rate_mult;
  56
  57                 ignore_nice = od_tuners->ignore_nice_load;
  58         } else {
  59                 sampling_rate = cs_tuners->sampling_rate;
  60                 ignore_nice = cs_tuners->ignore_nice_load;
  61         }
  62
  63         /* Get Absolute Load */
  64         for_each_cpu(j, policy->cpus) {
  65                 struct cpu_dbs_info *j_cdbs;
  66                 u64 cur_wall_time, cur_idle_time;
  67                 unsigned int idle_time, wall_time;
  68                 unsigned int load;
  69                 int io_busy = 0;
  70
  71                 j_cdbs = dbs_data->cdata->get_cpu_cdbs(j);
  72
  73                 /*
  74                  * For the purpose of ondemand, waiting for disk IO is
  75                  * an indication that you're performance critical, and
  76                  * not that the system is actually idle. So do not add
  77                  * the iowait time to the cpu idle time.
  78                  */
  79                 if (dbs_data->cdata->governor == GOV_ONDEMAND)
  80                         io_busy = od_tuners->io_is_busy;
  81                 cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy);
  82
  83                 wall_time = (unsigned int)
  84                         (cur_wall_time - j_cdbs->prev_cpu_wall);
  85                 j_cdbs->prev_cpu_wall = cur_wall_time;
  86
  87                 idle_time = (unsigned int)
  88                         (cur_idle_time - j_cdbs->prev_cpu_idle);
  89                 j_cdbs->prev_cpu_idle = cur_idle_time;
  90
  91                 if (ignore_nice) {
  92                         u64 cur_nice;
  93                         unsigned long cur_nice_jiffies;
  94
  95                         cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE] -
  96                                          cdbs->prev_cpu_nice;
  97                         /*
  98                          * Assumption: nice time between sampling periods will
  99                          * be less than 2^32 jiffies for 32 bit sys
 100                          */
 101                         cur_nice_jiffies = (unsigned long)
 102                                         cputime64_to_jiffies64(cur_nice);
 103
 104                         cdbs->prev_cpu_nice =
 105                                 kcpustat_cpu(j).cpustat[CPUTIME_NICE];
 106                         idle_time += jiffies_to_usecs(cur_nice_jiffies);
 107                 }
 108
 109                 if (unlikely(!wall_time || wall_time < idle_time))
 110                         continue;
 111
 112                 /*
 113                  * If the CPU had gone completely idle, and a task just woke up
 114                  * on this CPU now, it would be unfair to calculate 'load' the
 115                  * usual way for this elapsed time-window, because it will show
 116                  * near-zero load, irrespective of how CPU intensive that task
 117                  * actually is. This is undesirable for latency-sensitive bursty
 118                  * workloads.
 119                  *
 120                  * To avoid this, we reuse the 'load' from the previous
 121                  * time-window and give this task a chance to start with a
 122                  * reasonably high CPU frequency. (However, we shouldn't over-do
 123                  * this copy, lest we get stuck at a high load (high frequency)
 124                  * for too long, even when the current system load has actually
 125                  * dropped down. So we perform the copy only once, upon the
 126                  * first wake-up from idle.)
 127                  *
 128                  * Detecting this situation is easy: the governor's deferrable
 129                  * timer would not have fired during CPU-idle periods. Hence
 130                  * an unusually large 'wall_time' (as compared to the sampling
 131                  * rate) indicates this scenario.
 132                  *
 133                  * prev_load can be zero in two cases and we must recalculate it
 134                  * for both cases:
 135                  * - during long idle intervals
 136                  * - explicitly set to zero
 137                  */
 138                 if (unlikely(wall_time > (2 * sampling_rate) &&
 139                              j_cdbs->prev_load)) {
 140                         load = j_cdbs->prev_load;
 141
 142                         /*
 143                          * Perform a destructive copy, to ensure that we copy
 144                          * the previous load only once, upon the first wake-up
 145                          * from idle.
 146                          */
 147                         j_cdbs->prev_load = 0;
 148                 } else {
 149                         load = 100 * (wall_time - idle_time) / wall_time;
 150                         j_cdbs->prev_load = load;
 151                 }
 152
 153                 if (load > max_load)
 154                         max_load = load;
 155         }
 156
 157         dbs_data->cdata->gov_check_cpu(cpu, max_load);
 158 }
 159 EXPORT_SYMBOL_GPL(dbs_check_cpu);
 160
 161 static inline void __gov_queue_work(int cpu, struct dbs_data *dbs_data,
 162                 unsigned int delay)
 163 {
 164         struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
 165
 166         mod_delayed_work_on(cpu, system_wq, &cdbs->dwork, delay);
 167 }
 168
 169 void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy,
 170                 unsigned int delay, bool all_cpus)
 171 {
 172         int i;
 173
 174         mutex_lock(&cpufreq_governor_lock);
 175         if (!policy->governor_enabled)
 176                 goto out_unlock;
 177
 178         if (!all_cpus) {
 179                 /*
 180                  * Use raw_smp_processor_id() to avoid preemptible warnings.
 181                  * We know that this is only called with all_cpus == false from
 182                  * works that have been queued with *_work_on() functions and
 183                  * those works are canceled during CPU_DOWN_PREPARE so they
 184                  * can't possibly run on any other CPU.
 185                  */
 186                 __gov_queue_work(raw_smp_processor_id(), dbs_data, delay);
 187         } else {
 188                 for_each_cpu(i, policy->cpus)
 189                         __gov_queue_work(i, dbs_data, delay);
 190         }
 191
 192 out_unlock:
 193         mutex_unlock(&cpufreq_governor_lock);
 194 }
 195 EXPORT_SYMBOL_GPL(gov_queue_work);
 196
 197 static inline void gov_cancel_work(struct dbs_data *dbs_data,
 198                 struct cpufreq_policy *policy)
 199 {
 200         struct cpu_dbs_info *cdbs;
 201         int i;
 202
 203         for_each_cpu(i, policy->cpus) {
 204                 cdbs = dbs_data->cdata->get_cpu_cdbs(i);
 205                 cancel_delayed_work_sync(&cdbs->dwork);
 206         }
 207 }
 208
 209 /* Will return if we need to evaluate cpu load again or not */
 210 static bool need_load_eval(struct cpu_common_dbs_info *shared,
 211                            unsigned int sampling_rate)
 212 {
 213         if (policy_is_shared(shared->policy)) {
 214                 ktime_t time_now = ktime_get();
 215                 s64 delta_us = ktime_us_delta(time_now, shared->time_stamp);
 216
 217                 /* Do nothing if we recently have sampled */
 218                 if (delta_us < (s64)(sampling_rate / 2))
 219                         return false;
 220                 else
 221                         shared->time_stamp = time_now;
 222         }
 223
 224         return true;
 225 }
 226
 227 static void dbs_timer(struct work_struct *work)
 228 {
 229         struct cpu_dbs_info *cdbs = container_of(work, struct cpu_dbs_info,
 230                                                  dwork.work);
 231         struct cpu_common_dbs_info *shared = cdbs->shared;
 232         struct cpufreq_policy *policy = shared->policy;
 233         struct dbs_data *dbs_data = policy->governor_data;
 234         unsigned int sampling_rate, delay;
 235         bool modify_all = true;
 236
 237         mutex_lock(&shared->timer_mutex);
 238
 239         if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
 240                 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
 241
 242                 sampling_rate = cs_tuners->sampling_rate;
 243         } else {
 244                 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
 245
 246                 sampling_rate = od_tuners->sampling_rate;
 247         }
 248
 249         if (!need_load_eval(cdbs->shared, sampling_rate))
 250                 modify_all = false;
 251
 252         delay = dbs_data->cdata->gov_dbs_timer(cdbs, dbs_data, modify_all);
 253         gov_queue_work(dbs_data, policy, delay, modify_all);
 254
 255         mutex_unlock(&shared->timer_mutex);
 256 }
 257
 258 static void set_sampling_rate(struct dbs_data *dbs_data,
 259                 unsigned int sampling_rate)
 260 {
 261         if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
 262                 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
 263                 cs_tuners->sampling_rate = sampling_rate;
 264         } else {
 265                 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
 266                 od_tuners->sampling_rate = sampling_rate;
 267         }
 268 }
 269
 270 static int alloc_common_dbs_info(struct cpufreq_policy *policy,
 271                                  struct common_dbs_data *cdata)
 272 {
 273         struct cpu_common_dbs_info *shared;
 274         int j;
 275
 276         /* Allocate memory for the common information for policy->cpus */
 277         shared = kzalloc(sizeof(*shared), GFP_KERNEL);
 278         if (!shared)
 279                 return -ENOMEM;
 280
 281         /* Set shared for all CPUs, online+offline */
 282         for_each_cpu(j, policy->related_cpus)
 283                 cdata->get_cpu_cdbs(j)->shared = shared;
 284
 285         return 0;
 286 }
 287
 288 static void free_common_dbs_info(struct cpufreq_policy *policy,
 289                                  struct common_dbs_data *cdata)
 290 {
 291         struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu);
 292         struct cpu_common_dbs_info *shared = cdbs->shared;
 293         int j;
 294
 295         for_each_cpu(j, policy->cpus)
 296                 cdata->get_cpu_cdbs(j)->shared = NULL;
 297
 298         kfree(shared);
 299 }
 300
 301 static int cpufreq_governor_init(struct cpufreq_policy *policy,
 302                                  struct dbs_data *dbs_data,
 303                                  struct common_dbs_data *cdata)
 304 {
 305         unsigned int latency;
 306         int ret;
 307
 308         /* State should be equivalent to EXIT */
 309         if (policy->governor_data)
 310                 return -EBUSY;
 311
 312         if (dbs_data) {
 313                 if (WARN_ON(have_governor_per_policy()))
 314                         return -EINVAL;
 315
 316                 ret = alloc_common_dbs_info(policy, cdata);
 317                 if (ret)
 318                         return ret;
 319
 320                 dbs_data->usage_count++;
 321                 policy->governor_data = dbs_data;
 322                 return 0;
 323         }
 324
 325         dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
 326         if (!dbs_data)
 327                 return -ENOMEM;
 328
 329         ret = alloc_common_dbs_info(policy, cdata);
 330         if (ret)
 331                 goto free_dbs_data;
 332
 333         dbs_data->cdata = cdata;
 334         dbs_data->usage_count = 1;
 335
 336         ret = cdata->init(dbs_data, !policy->governor->initialized);
 337         if (ret)
 338                 goto free_common_dbs_info;
 339
 340         /* policy latency is in ns. Convert it to us first */
 341         latency = policy->cpuinfo.transition_latency / 1000;
 342         if (latency == 0)
 343                 latency = 1;
 344
 345         /* Bring kernel and HW constraints together */
 346         dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
 347                                           MIN_LATENCY_MULTIPLIER * latency);
 348         set_sampling_rate(dbs_data, max(dbs_data->min_sampling_rate,
 349                                         latency * LATENCY_MULTIPLIER));
 350
 351         if (!have_governor_per_policy())
 352                 cdata->gdbs_data = dbs_data;
 353
 354         ret = sysfs_create_group(get_governor_parent_kobj(policy),
 355                                  get_sysfs_attr(dbs_data));
 356         if (ret)
 357                 goto reset_gdbs_data;
 358
 359         policy->governor_data = dbs_data;
 360
 361         return 0;
 362
 363 reset_gdbs_data:
 364         if (!have_governor_per_policy())
 365                 cdata->gdbs_data = NULL;
 366         cdata->exit(dbs_data, !policy->governor->initialized);
 367 free_common_dbs_info:
 368         free_common_dbs_info(policy, cdata);
 369 free_dbs_data:
 370         kfree(dbs_data);
 371         return ret;
 372 }
 373
 374 static int cpufreq_governor_exit(struct cpufreq_policy *policy,
 375                                  struct dbs_data *dbs_data)
 376 {
 377         struct common_dbs_data *cdata = dbs_data->cdata;
 378         struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu);
 379
 380         /* State should be equivalent to INIT */
 381         if (!cdbs->shared || cdbs->shared->policy)
 382                 return -EBUSY;
 383
 384         policy->governor_data = NULL;
 385         if (!--dbs_data->usage_count) {
 386                 sysfs_remove_group(get_governor_parent_kobj(policy),
 387                                    get_sysfs_attr(dbs_data));
 388
 389                 if (!have_governor_per_policy())
 390                         cdata->gdbs_data = NULL;
 391
 392                 cdata->exit(dbs_data, policy->governor->initialized == 1);
 393                 kfree(dbs_data);
 394         }
 395
 396         free_common_dbs_info(policy, cdata);
 397         return 0;
 398 }
 399
 400 static int cpufreq_governor_start(struct cpufreq_policy *policy,
 401                                   struct dbs_data *dbs_data)
 402 {
 403         struct common_dbs_data *cdata = dbs_data->cdata;
 404         unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu;
 405         struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu);
 406         struct cpu_common_dbs_info *shared = cdbs->shared;
 407         int io_busy = 0;
 408
 409         if (!policy->cur)
 410                 return -EINVAL;
 411
 412         /* State should be equivalent to INIT */
 413         if (!shared || shared->policy)
 414                 return -EBUSY;
 415
 416         if (cdata->governor == GOV_CONSERVATIVE) {
 417                 struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
 418
 419                 sampling_rate = cs_tuners->sampling_rate;
 420                 ignore_nice = cs_tuners->ignore_nice_load;
 421         } else {
 422                 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
 423
 424                 sampling_rate = od_tuners->sampling_rate;
 425                 ignore_nice = od_tuners->ignore_nice_load;
 426                 io_busy = od_tuners->io_is_busy;
 427         }
 428
 429         shared->policy = policy;
 430         shared->time_stamp = ktime_get();
 431         mutex_init(&shared->timer_mutex);
 432
 433         for_each_cpu(j, policy->cpus) {
 434                 struct cpu_dbs_info *j_cdbs = cdata->get_cpu_cdbs(j);
 435                 unsigned int prev_load;
 436
 437                 j_cdbs->prev_cpu_idle =
 438                         get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);
 439
 440                 prev_load = (unsigned int)(j_cdbs->prev_cpu_wall -
 441                                             j_cdbs->prev_cpu_idle);
 442                 j_cdbs->prev_load = 100 * prev_load /
 443                                     (unsigned int)j_cdbs->prev_cpu_wall;
 444
 445                 if (ignore_nice)
 446                         j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
 447
 448                 INIT_DEFERRABLE_WORK(&j_cdbs->dwork, dbs_timer);
 449         }
 450
 451         if (cdata->governor == GOV_CONSERVATIVE) {
 452                 struct cs_cpu_dbs_info_s *cs_dbs_info =
 453                         cdata->get_cpu_dbs_info_s(cpu);
 454
 455                 cs_dbs_info->down_skip = 0;
 456                 cs_dbs_info->requested_freq = policy->cur;
 457         } else {
 458                 struct od_ops *od_ops = cdata->gov_ops;
 459                 struct od_cpu_dbs_info_s *od_dbs_info = cdata->get_cpu_dbs_info_s(cpu);
 460
 461                 od_dbs_info->rate_mult = 1;
 462                 od_dbs_info->sample_type = OD_NORMAL_SAMPLE;
 463                 od_ops->powersave_bias_init_cpu(cpu);
 464         }
 465
 466         gov_queue_work(dbs_data, policy, delay_for_sampling_rate(sampling_rate),
 467                        true);
 468         return 0;
 469 }
 470
 471 static int cpufreq_governor_stop(struct cpufreq_policy *policy,
 472                                  struct dbs_data *dbs_data)
 473 {
 474         struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(policy->cpu);
 475         struct cpu_common_dbs_info *shared = cdbs->shared;
 476
 477         /* State should be equivalent to START */
 478         if (!shared || !shared->policy)
 479                 return -EBUSY;
 480
 481         gov_cancel_work(dbs_data, policy);
 482
 483         shared->policy = NULL;
 484         mutex_destroy(&shared->timer_mutex);
 485         return 0;
 486 }
 487
 488 static int cpufreq_governor_limits(struct cpufreq_policy *policy,
 489                                    struct dbs_data *dbs_data)
 490 {
 491         struct common_dbs_data *cdata = dbs_data->cdata;
 492         unsigned int cpu = policy->cpu;
 493         struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu);
 494
 495         /* State should be equivalent to START */
 496         if (!cdbs->shared || !cdbs->shared->policy)
 497                 return -EBUSY;
 498
 499         mutex_lock(&cdbs->shared->timer_mutex);
 500         if (policy->max < cdbs->shared->policy->cur)
 501                 __cpufreq_driver_target(cdbs->shared->policy, policy->max,
 502                                         CPUFREQ_RELATION_H);
 503         else if (policy->min > cdbs->shared->policy->cur)
 504                 __cpufreq_driver_target(cdbs->shared->policy, policy->min,
 505                                         CPUFREQ_RELATION_L);
 506         dbs_check_cpu(dbs_data, cpu);
 507         mutex_unlock(&cdbs->shared->timer_mutex);
 508
 509         return 0;
 510 }
 511
 512 int cpufreq_governor_dbs(struct cpufreq_policy *policy,
 513                          struct common_dbs_data *cdata, unsigned int event)
 514 {
 515         struct dbs_data *dbs_data;
 516         int ret;
 517
 518         /* Lock governor to block concurrent initialization of governor */
 519         mutex_lock(&cdata->mutex);
 520
 521         if (have_governor_per_policy())
 522                 dbs_data = policy->governor_data;
 523         else
 524                 dbs_data = cdata->gdbs_data;
 525
 526         if (!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT)) {
 527                 ret = -EINVAL;
 528                 goto unlock;
 529         }
 530
 531         switch (event) {
 532         case CPUFREQ_GOV_POLICY_INIT:
 533                 ret = cpufreq_governor_init(policy, dbs_data, cdata);
 534                 break;
 535         case CPUFREQ_GOV_POLICY_EXIT:
 536                 ret = cpufreq_governor_exit(policy, dbs_data);
 537                 break;
 538         case CPUFREQ_GOV_START:
 539                 ret = cpufreq_governor_start(policy, dbs_data);
 540                 break;
 541         case CPUFREQ_GOV_STOP:
 542                 ret = cpufreq_governor_stop(policy, dbs_data);
 543                 break;
 544         case CPUFREQ_GOV_LIMITS:
 545                 ret = cpufreq_governor_limits(policy, dbs_data);
 546                 break;
 547         default:
 548                 ret = -EINVAL;
 549         }
 550
 551 unlock:
 552         mutex_unlock(&cdata->mutex);
 553
 554         return ret;
 555 }
 556 EXPORT_SYMBOL_GPL(cpufreq_governor_dbs);