block/blk-cgroup.h

   1 #ifndef _BLK_CGROUP_H
   2 #define _BLK_CGROUP_H
   3 /*
   4  * Common Block IO controller cgroup interface
   5  *
   6  * Based on ideas and code from CFQ, CFS and BFQ:
   7  * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
   8  *
   9  * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  10  *                    Paolo Valente <paolo.valente@unimore.it>
  11  *
  12  * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  13  *                    Nauman Rafique <nauman@google.com>
  14  */
  15
  16 #include <linux/cgroup.h>
  17 #include <linux/u64_stats_sync.h>
  18 #include <linux/seq_file.h>
  19
  20 enum blkio_policy_id {
  21         BLKIO_POLICY_PROP = 0,          /* Proportional Bandwidth division */
  22         BLKIO_POLICY_THROTL,            /* Throttling */
  23
  24         BLKIO_NR_POLICIES,
  25 };
  26
  27 /* Max limits for throttle policy */
  28 #define THROTL_IOPS_MAX         UINT_MAX
  29
  30 #ifdef CONFIG_BLK_CGROUP
  31
  32 /* cft->private [un]packing for stat printing */
  33 #define BLKCG_STAT_PRIV(pol, off)       (((unsigned)(pol) << 16) | (off))
  34 #define BLKCG_STAT_POL(prv)             ((unsigned)(prv) >> 16)
  35 #define BLKCG_STAT_OFF(prv)             ((unsigned)(prv) & 0xffff)
  36
  37 enum blkg_rwstat_type {
  38         BLKG_RWSTAT_READ,
  39         BLKG_RWSTAT_WRITE,
  40         BLKG_RWSTAT_SYNC,
  41         BLKG_RWSTAT_ASYNC,
  42
  43         BLKG_RWSTAT_NR,
  44         BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
  45 };
  46
  47 struct blkio_cgroup {
  48         struct cgroup_subsys_state css;
  49         unsigned int weight;
  50         spinlock_t lock;
  51         struct hlist_head blkg_list;
  52
  53         /* for policies to test whether associated blkcg has changed */
  54         uint64_t id;
  55 };
  56
  57 struct blkg_stat {
  58         struct u64_stats_sync           syncp;
  59         uint64_t                        cnt;
  60 };
  61
  62 struct blkg_rwstat {
  63         struct u64_stats_sync           syncp;
  64         uint64_t                        cnt[BLKG_RWSTAT_NR];
  65 };
  66
  67 struct blkio_group_stats {
  68         /* total bytes transferred */
  69         struct blkg_rwstat              service_bytes;
  70         /* total IOs serviced, post merge */
  71         struct blkg_rwstat              serviced;
  72         /* number of ios merged */
  73         struct blkg_rwstat              merged;
  74         /* total time spent on device in ns, may not be accurate w/ queueing */
  75         struct blkg_rwstat              service_time;
  76         /* total time spent waiting in scheduler queue in ns */
  77         struct blkg_rwstat              wait_time;
  78         /* number of IOs queued up */
  79         struct blkg_rwstat              queued;
  80         /* total sectors transferred */
  81         struct blkg_stat                sectors;
  82         /* total disk time and nr sectors dispatched by this group */
  83         struct blkg_stat                time;
  84 #ifdef CONFIG_DEBUG_BLK_CGROUP
  85         /* time not charged to this cgroup */
  86         struct blkg_stat                unaccounted_time;
  87         /* sum of number of ios queued across all samples */
  88         struct blkg_stat                avg_queue_size_sum;
  89         /* count of samples taken for average */
  90         struct blkg_stat                avg_queue_size_samples;
  91         /* how many times this group has been removed from service tree */
  92         struct blkg_stat                dequeue;
  93         /* total time spent waiting for it to be assigned a timeslice. */
  94         struct blkg_stat                group_wait_time;
  95         /* time spent idling for this blkio_group */
  96         struct blkg_stat                idle_time;
  97         /* total time with empty current active q with other requests queued */
  98         struct blkg_stat                empty_time;
  99         /* fields after this shouldn't be cleared on stat reset */
 100         uint64_t                        start_group_wait_time;
 101         uint64_t                        start_idle_time;
 102         uint64_t                        start_empty_time;
 103         uint16_t                        flags;
 104 #endif
 105 };
 106
 107 /* Per cpu blkio group stats */
 108 struct blkio_group_stats_cpu {
 109         /* total bytes transferred */
 110         struct blkg_rwstat              service_bytes;
 111         /* total IOs serviced, post merge */
 112         struct blkg_rwstat              serviced;
 113 };
 114
 115 struct blkio_group_conf {
 116         unsigned int weight;
 117         u64 iops[2];
 118         u64 bps[2];
 119 };
 120
 121 /* per-blkg per-policy data */
 122 struct blkg_policy_data {
 123         /* the blkg this per-policy data belongs to */
 124         struct blkio_group *blkg;
 125
 126         /* Configuration */
 127         struct blkio_group_conf conf;
 128
 129         struct blkio_group_stats stats;
 130         /* Per cpu stats pointer */
 131         struct blkio_group_stats_cpu __percpu *stats_cpu;
 132
 133         /* pol->pdata_size bytes of private data used by policy impl */
 134         char pdata[] __aligned(__alignof__(unsigned long long));
 135 };
 136
 137 struct blkio_group {
 138         /* Pointer to the associated request_queue */
 139         struct request_queue *q;
 140         struct list_head q_node;
 141         struct hlist_node blkcg_node;
 142         struct blkio_cgroup *blkcg;
 143         /* Store cgroup path */
 144         char path[128];
 145         /* reference count */
 146         int refcnt;
 147
 148         struct blkg_policy_data *pd[BLKIO_NR_POLICIES];
 149
 150         /* List of blkg waiting for per cpu stats memory to be allocated */
 151         struct list_head alloc_node;
 152         struct rcu_head rcu_head;
 153 };
 154
 155 typedef void (blkio_init_group_fn)(struct blkio_group *blkg);
 156
 157 struct blkio_policy_ops {
 158         blkio_init_group_fn *blkio_init_group_fn;
 159 };
 160
 161 struct blkio_policy_type {
 162         struct list_head list;
 163         struct blkio_policy_ops ops;
 164         enum blkio_policy_id plid;
 165         size_t pdata_size;              /* policy specific private data size */
 166         struct cftype *cftypes;         /* cgroup files for the policy */
 167 };
 168
 169 extern int blkcg_init_queue(struct request_queue *q);
 170 extern void blkcg_drain_queue(struct request_queue *q);
 171 extern void blkcg_exit_queue(struct request_queue *q);
 172
 173 /* Blkio controller policy registration */
 174 extern void blkio_policy_register(struct blkio_policy_type *);
 175 extern void blkio_policy_unregister(struct blkio_policy_type *);
 176 extern void blkg_destroy_all(struct request_queue *q, bool destroy_root);
 177 extern void update_root_blkg_pd(struct request_queue *q,
 178                                 enum blkio_policy_id plid);
 179
 180 void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg,
 181                        u64 (*prfill)(struct seq_file *, struct blkg_policy_data *, int),
 182                        int pol, int data, bool show_total);
 183 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
 184 u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 185                          const struct blkg_rwstat *rwstat);
 186 int blkcg_print_stat(struct cgroup *cgrp, struct cftype *cft,
 187                      struct seq_file *sf);
 188 int blkcg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
 189                        struct seq_file *sf);
 190
 191 struct blkg_conf_ctx {
 192         struct gendisk          *disk;
 193         struct blkio_group      *blkg;
 194         u64                     v;
 195 };
 196
 197 int blkg_conf_prep(struct blkio_cgroup *blkcg, const char *input,
 198                    struct blkg_conf_ctx *ctx);
 199 void blkg_conf_finish(struct blkg_conf_ctx *ctx);
 200
 201
 202 /**
 203  * blkg_to_pdata - get policy private data
 204  * @blkg: blkg of interest
 205  * @pol: policy of interest
 206  *
 207  * Return pointer to private data associated with the @blkg-@pol pair.
 208  */
 209 static inline void *blkg_to_pdata(struct blkio_group *blkg,
 210                               struct blkio_policy_type *pol)
 211 {
 212         return blkg ? blkg->pd[pol->plid]->pdata : NULL;
 213 }
 214
 215 /**
 216  * pdata_to_blkg - get blkg associated with policy private data
 217  * @pdata: policy private data of interest
 218  *
 219  * @pdata is policy private data.  Determine the blkg it's associated with.
 220  */
 221 static inline struct blkio_group *pdata_to_blkg(void *pdata)
 222 {
 223         if (pdata) {
 224                 struct blkg_policy_data *pd =
 225                         container_of(pdata, struct blkg_policy_data, pdata);
 226                 return pd->blkg;
 227         }
 228         return NULL;
 229 }
 230
 231 static inline char *blkg_path(struct blkio_group *blkg)
 232 {
 233         return blkg->path;
 234 }
 235
 236 /**
 237  * blkg_get - get a blkg reference
 238  * @blkg: blkg to get
 239  *
 240  * The caller should be holding queue_lock and an existing reference.
 241  */
 242 static inline void blkg_get(struct blkio_group *blkg)
 243 {
 244         lockdep_assert_held(blkg->q->queue_lock);
 245         WARN_ON_ONCE(!blkg->refcnt);
 246         blkg->refcnt++;
 247 }
 248
 249 void __blkg_release(struct blkio_group *blkg);
 250
 251 /**
 252  * blkg_put - put a blkg reference
 253  * @blkg: blkg to put
 254  *
 255  * The caller should be holding queue_lock.
 256  */
 257 static inline void blkg_put(struct blkio_group *blkg)
 258 {
 259         lockdep_assert_held(blkg->q->queue_lock);
 260         WARN_ON_ONCE(blkg->refcnt <= 0);
 261         if (!--blkg->refcnt)
 262                 __blkg_release(blkg);
 263 }
 264
 265 /**
 266  * blkg_stat_add - add a value to a blkg_stat
 267  * @stat: target blkg_stat
 268  * @val: value to add
 269  *
 270  * Add @val to @stat.  The caller is responsible for synchronizing calls to
 271  * this function.
 272  */
 273 static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
 274 {
 275         u64_stats_update_begin(&stat->syncp);
 276         stat->cnt += val;
 277         u64_stats_update_end(&stat->syncp);
 278 }
 279
 280 /**
 281  * blkg_stat_read - read the current value of a blkg_stat
 282  * @stat: blkg_stat to read
 283  *
 284  * Read the current value of @stat.  This function can be called without
 285  * synchroniztion and takes care of u64 atomicity.
 286  */
 287 static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
 288 {
 289         unsigned int start;
 290         uint64_t v;
 291
 292         do {
 293                 start = u64_stats_fetch_begin(&stat->syncp);
 294                 v = stat->cnt;
 295         } while (u64_stats_fetch_retry(&stat->syncp, start));
 296
 297         return v;
 298 }
 299
 300 /**
 301  * blkg_stat_reset - reset a blkg_stat
 302  * @stat: blkg_stat to reset
 303  */
 304 static inline void blkg_stat_reset(struct blkg_stat *stat)
 305 {
 306         stat->cnt = 0;
 307 }
 308
 309 /**
 310  * blkg_rwstat_add - add a value to a blkg_rwstat
 311  * @rwstat: target blkg_rwstat
 312  * @rw: mask of REQ_{WRITE|SYNC}
 313  * @val: value to add
 314  *
 315  * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 316  * caller is responsible for synchronizing calls to this function.
 317  */
 318 static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
 319                                    int rw, uint64_t val)
 320 {
 321         u64_stats_update_begin(&rwstat->syncp);
 322
 323         if (rw & REQ_WRITE)
 324                 rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
 325         else
 326                 rwstat->cnt[BLKG_RWSTAT_READ] += val;
 327         if (rw & REQ_SYNC)
 328                 rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
 329         else
 330                 rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;
 331
 332         u64_stats_update_end(&rwstat->syncp);
 333 }
 334
 335 /**
 336  * blkg_rwstat_read - read the current values of a blkg_rwstat
 337  * @rwstat: blkg_rwstat to read
 338  *
 339  * Read the current snapshot of @rwstat and return it as the return value.
 340  * This function can be called without synchronization and takes care of
 341  * u64 atomicity.
 342  */
 343 static struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
 344 {
 345         unsigned int start;
 346         struct blkg_rwstat tmp;
 347
 348         do {
 349                 start = u64_stats_fetch_begin(&rwstat->syncp);
 350                 tmp = *rwstat;
 351         } while (u64_stats_fetch_retry(&rwstat->syncp, start));
 352
 353         return tmp;
 354 }
 355
 356 /**
 357  * blkg_rwstat_sum - read the total count of a blkg_rwstat
 358  * @rwstat: blkg_rwstat to read
 359  *
 360  * Return the total count of @rwstat regardless of the IO direction.  This
 361  * function can be called without synchronization and takes care of u64
 362  * atomicity.
 363  */
 364 static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat)
 365 {
 366         struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
 367
 368         return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
 369 }
 370
 371 /**
 372  * blkg_rwstat_reset - reset a blkg_rwstat
 373  * @rwstat: blkg_rwstat to reset
 374  */
 375 static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
 376 {
 377         memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
 378 }
 379
 380 #else
 381
 382 struct blkio_group {
 383 };
 384
 385 struct blkio_policy_type {
 386 };
 387
 388 static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
 389 static inline void blkcg_drain_queue(struct request_queue *q) { }
 390 static inline void blkcg_exit_queue(struct request_queue *q) { }
 391 static inline void blkio_policy_register(struct blkio_policy_type *blkiop) { }
 392 static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { }
 393 static inline void blkg_destroy_all(struct request_queue *q,
 394                                     bool destory_root) { }
 395 static inline void update_root_blkg_pd(struct request_queue *q,
 396                                        enum blkio_policy_id plid) { }
 397
 398 static inline void *blkg_to_pdata(struct blkio_group *blkg,
 399                                 struct blkio_policy_type *pol) { return NULL; }
 400 static inline struct blkio_group *pdata_to_blkg(void *pdata,
 401                                 struct blkio_policy_type *pol) { return NULL; }
 402 static inline char *blkg_path(struct blkio_group *blkg) { return NULL; }
 403 static inline void blkg_get(struct blkio_group *blkg) { }
 404 static inline void blkg_put(struct blkio_group *blkg) { }
 405
 406 #endif
 407
 408 #define BLKIO_WEIGHT_MIN        10
 409 #define BLKIO_WEIGHT_MAX        1000
 410 #define BLKIO_WEIGHT_DEFAULT    500
 411
 412 #ifdef CONFIG_BLK_CGROUP
 413 extern struct blkio_cgroup blkio_root_cgroup;
 414 extern struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup);
 415 extern struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio);
 416 extern struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
 417                                        struct request_queue *q);
 418 struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
 419                                        struct request_queue *q,
 420                                        bool for_root);
 421 #else
 422 struct cgroup;
 423 static inline struct blkio_cgroup *
 424 cgroup_to_blkio_cgroup(struct cgroup *cgroup) { return NULL; }
 425 static inline struct blkio_cgroup *
 426 bio_blkio_cgroup(struct bio *bio) { return NULL; }
 427
 428 static inline struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
 429                                               void *key) { return NULL; }
 430 #endif
 431 #endif /* _BLK_CGROUP_H */