block/blk-cgroup.h

   1 #ifndef _BLK_CGROUP_H
   2 #define _BLK_CGROUP_H
   3 /*
   4  * Common Block IO controller cgroup interface
   5  *
   6  * Based on ideas and code from CFQ, CFS and BFQ:
   7  * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
   8  *
   9  * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  10  *                    Paolo Valente <paolo.valente@unimore.it>
  11  *
  12  * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  13  *                    Nauman Rafique <nauman@google.com>
  14  */
  15
  16 #include <linux/cgroup.h>
  17 #include <linux/u64_stats_sync.h>
  18 #include <linux/seq_file.h>
  19
  20 /* Max limits for throttle policy */
  21 #define THROTL_IOPS_MAX         UINT_MAX
  22
  23 /* CFQ specific, out here for blkcg->cfq_weight */
  24 #define CFQ_WEIGHT_MIN          10
  25 #define CFQ_WEIGHT_MAX          1000
  26 #define CFQ_WEIGHT_DEFAULT      500
  27
  28 #ifdef CONFIG_BLK_CGROUP
  29
  30 enum blkg_rwstat_type {
  31         BLKG_RWSTAT_READ,
  32         BLKG_RWSTAT_WRITE,
  33         BLKG_RWSTAT_SYNC,
  34         BLKG_RWSTAT_ASYNC,
  35
  36         BLKG_RWSTAT_NR,
  37         BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
  38 };
  39
  40 struct blkcg {
  41         struct cgroup_subsys_state      css;
  42         spinlock_t                      lock;
  43         struct hlist_head               blkg_list;
  44
  45         /* for policies to test whether associated blkcg has changed */
  46         uint64_t                        id;
  47
  48         /* TODO: per-policy storage in blkcg */
  49         unsigned int                    cfq_weight;     /* belongs to cfq */
  50 };
  51
  52 struct blkg_stat {
  53         struct u64_stats_sync           syncp;
  54         uint64_t                        cnt;
  55 };
  56
  57 struct blkg_rwstat {
  58         struct u64_stats_sync           syncp;
  59         uint64_t                        cnt[BLKG_RWSTAT_NR];
  60 };
  61
  62 /* per-blkg per-policy data */
  63 struct blkg_policy_data {
  64         /* the blkg this per-policy data belongs to */
  65         struct blkcg_gq                 *blkg;
  66
  67         /* used during policy activation */
  68         struct list_head                alloc_node;
  69
  70         /* pol->pdata_size bytes of private data used by policy impl */
  71         char                            pdata[] __aligned(__alignof__(unsigned long long));
  72 };
  73
  74 /* association between a blk cgroup and a request queue */
  75 struct blkcg_gq {
  76         /* Pointer to the associated request_queue */
  77         struct request_queue            *q;
  78         struct list_head                q_node;
  79         struct hlist_node               blkcg_node;
  80         struct blkcg                    *blkcg;
  81         /* reference count */
  82         int                             refcnt;
  83
  84         struct blkg_policy_data         *pd[BLKCG_MAX_POLS];
  85
  86         struct rcu_head                 rcu_head;
  87 };
  88
  89 typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg);
  90 typedef void (blkcg_pol_exit_pd_fn)(struct blkcg_gq *blkg);
  91 typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkcg_gq *blkg);
  92
  93 struct blkcg_policy_ops {
  94         blkcg_pol_init_pd_fn            *pd_init_fn;
  95         blkcg_pol_exit_pd_fn            *pd_exit_fn;
  96         blkcg_pol_reset_pd_stats_fn     *pd_reset_stats_fn;
  97 };
  98
  99 struct blkcg_policy {
 100         struct blkcg_policy_ops         ops;
 101         int                             plid;
 102         /* policy specific private data size */
 103         size_t                          pdata_size;
 104         /* cgroup files for the policy */
 105         struct cftype                   *cftypes;
 106 };
 107
 108 extern struct blkcg blkcg_root;
 109
 110 struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup);
 111 struct blkcg *bio_blkcg(struct bio *bio);
 112 struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q);
 113 struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
 114                                     struct request_queue *q);
 115 int blkcg_init_queue(struct request_queue *q);
 116 void blkcg_drain_queue(struct request_queue *q);
 117 void blkcg_exit_queue(struct request_queue *q);
 118
 119 /* Blkio controller policy registration */
 120 int blkcg_policy_register(struct blkcg_policy *pol);
 121 void blkcg_policy_unregister(struct blkcg_policy *pol);
 122 int blkcg_activate_policy(struct request_queue *q,
 123                           const struct blkcg_policy *pol);
 124 void blkcg_deactivate_policy(struct request_queue *q,
 125                              const struct blkcg_policy *pol);
 126
 127 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
 128                        u64 (*prfill)(struct seq_file *, void *, int),
 129                        const struct blkcg_policy *pol, int data,
 130                        bool show_total);
 131 u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v);
 132 u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
 133                          const struct blkg_rwstat *rwstat);
 134 u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off);
 135 u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off);
 136
 137 struct blkg_conf_ctx {
 138         struct gendisk                  *disk;
 139         struct blkcg_gq                 *blkg;
 140         u64                             v;
 141 };
 142
 143 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
 144                    const char *input, struct blkg_conf_ctx *ctx);
 145 void blkg_conf_finish(struct blkg_conf_ctx *ctx);
 146
 147
 148 /**
 149  * blkg_to_pdata - get policy private data
 150  * @blkg: blkg of interest
 151  * @pol: policy of interest
 152  *
 153  * Return pointer to private data associated with the @blkg-@pol pair.
 154  */
 155 static inline void *blkg_to_pdata(struct blkcg_gq *blkg,
 156                                   struct blkcg_policy *pol)
 157 {
 158         return blkg ? blkg->pd[pol->plid]->pdata : NULL;
 159 }
 160
 161 /**
 162  * pdata_to_blkg - get blkg associated with policy private data
 163  * @pdata: policy private data of interest
 164  *
 165  * @pdata is policy private data.  Determine the blkg it's associated with.
 166  */
 167 static inline struct blkcg_gq *pdata_to_blkg(void *pdata)
 168 {
 169         if (pdata) {
 170                 struct blkg_policy_data *pd =
 171                         container_of(pdata, struct blkg_policy_data, pdata);
 172                 return pd->blkg;
 173         }
 174         return NULL;
 175 }
 176
 177 /**
 178  * blkg_path - format cgroup path of blkg
 179  * @blkg: blkg of interest
 180  * @buf: target buffer
 181  * @buflen: target buffer length
 182  *
 183  * Format the path of the cgroup of @blkg into @buf.
 184  */
 185 static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
 186 {
 187         int ret;
 188
 189         rcu_read_lock();
 190         ret = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
 191         rcu_read_unlock();
 192         if (ret)
 193                 strncpy(buf, "<unavailable>", buflen);
 194         return ret;
 195 }
 196
 197 /**
 198  * blkg_get - get a blkg reference
 199  * @blkg: blkg to get
 200  *
 201  * The caller should be holding queue_lock and an existing reference.
 202  */
 203 static inline void blkg_get(struct blkcg_gq *blkg)
 204 {
 205         lockdep_assert_held(blkg->q->queue_lock);
 206         WARN_ON_ONCE(!blkg->refcnt);
 207         blkg->refcnt++;
 208 }
 209
 210 void __blkg_release(struct blkcg_gq *blkg);
 211
 212 /**
 213  * blkg_put - put a blkg reference
 214  * @blkg: blkg to put
 215  *
 216  * The caller should be holding queue_lock.
 217  */
 218 static inline void blkg_put(struct blkcg_gq *blkg)
 219 {
 220         lockdep_assert_held(blkg->q->queue_lock);
 221         WARN_ON_ONCE(blkg->refcnt <= 0);
 222         if (!--blkg->refcnt)
 223                 __blkg_release(blkg);
 224 }
 225
 226 /**
 227  * blkg_stat_add - add a value to a blkg_stat
 228  * @stat: target blkg_stat
 229  * @val: value to add
 230  *
 231  * Add @val to @stat.  The caller is responsible for synchronizing calls to
 232  * this function.
 233  */
 234 static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
 235 {
 236         u64_stats_update_begin(&stat->syncp);
 237         stat->cnt += val;
 238         u64_stats_update_end(&stat->syncp);
 239 }
 240
 241 /**
 242  * blkg_stat_read - read the current value of a blkg_stat
 243  * @stat: blkg_stat to read
 244  *
 245  * Read the current value of @stat.  This function can be called without
 246  * synchroniztion and takes care of u64 atomicity.
 247  */
 248 static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
 249 {
 250         unsigned int start;
 251         uint64_t v;
 252
 253         do {
 254                 start = u64_stats_fetch_begin(&stat->syncp);
 255                 v = stat->cnt;
 256         } while (u64_stats_fetch_retry(&stat->syncp, start));
 257
 258         return v;
 259 }
 260
 261 /**
 262  * blkg_stat_reset - reset a blkg_stat
 263  * @stat: blkg_stat to reset
 264  */
 265 static inline void blkg_stat_reset(struct blkg_stat *stat)
 266 {
 267         stat->cnt = 0;
 268 }
 269
 270 /**
 271  * blkg_rwstat_add - add a value to a blkg_rwstat
 272  * @rwstat: target blkg_rwstat
 273  * @rw: mask of REQ_{WRITE|SYNC}
 274  * @val: value to add
 275  *
 276  * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 277  * caller is responsible for synchronizing calls to this function.
 278  */
 279 static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
 280                                    int rw, uint64_t val)
 281 {
 282         u64_stats_update_begin(&rwstat->syncp);
 283
 284         if (rw & REQ_WRITE)
 285                 rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
 286         else
 287                 rwstat->cnt[BLKG_RWSTAT_READ] += val;
 288         if (rw & REQ_SYNC)
 289                 rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
 290         else
 291                 rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;
 292
 293         u64_stats_update_end(&rwstat->syncp);
 294 }
 295
 296 /**
 297  * blkg_rwstat_read - read the current values of a blkg_rwstat
 298  * @rwstat: blkg_rwstat to read
 299  *
 300  * Read the current snapshot of @rwstat and return it as the return value.
 301  * This function can be called without synchronization and takes care of
 302  * u64 atomicity.
 303  */
 304 static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
 305 {
 306         unsigned int start;
 307         struct blkg_rwstat tmp;
 308
 309         do {
 310                 start = u64_stats_fetch_begin(&rwstat->syncp);
 311                 tmp = *rwstat;
 312         } while (u64_stats_fetch_retry(&rwstat->syncp, start));
 313
 314         return tmp;
 315 }
 316
 317 /**
 318  * blkg_rwstat_sum - read the total count of a blkg_rwstat
 319  * @rwstat: blkg_rwstat to read
 320  *
 321  * Return the total count of @rwstat regardless of the IO direction.  This
 322  * function can be called without synchronization and takes care of u64
 323  * atomicity.
 324  */
 325 static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat)
 326 {
 327         struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
 328
 329         return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
 330 }
 331
 332 /**
 333  * blkg_rwstat_reset - reset a blkg_rwstat
 334  * @rwstat: blkg_rwstat to reset
 335  */
 336 static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
 337 {
 338         memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
 339 }
 340
 341 #else   /* CONFIG_BLK_CGROUP */
 342
 343 struct cgroup;
 344
 345 struct blkcg_gq {
 346 };
 347
 348 struct blkcg_policy {
 349 };
 350
 351 static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; }
 352 static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
 353 static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
 354 static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
 355 static inline void blkcg_drain_queue(struct request_queue *q) { }
 356 static inline void blkcg_exit_queue(struct request_queue *q) { }
 357 static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
 358 static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
 359 static inline int blkcg_activate_policy(struct request_queue *q,
 360                                         const struct blkcg_policy *pol) { return 0; }
 361 static inline void blkcg_deactivate_policy(struct request_queue *q,
 362                                            const struct blkcg_policy *pol) { }
 363
 364 static inline void *blkg_to_pdata(struct blkcg_gq *blkg,
 365                                   struct blkcg_policy *pol) { return NULL; }
 366 static inline struct blkcg_gq *pdata_to_blkg(void *pdata,
 367                                   struct blkcg_policy *pol) { return NULL; }
 368 static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
 369 static inline void blkg_get(struct blkcg_gq *blkg) { }
 370 static inline void blkg_put(struct blkcg_gq *blkg) { }
 371
 372 #endif  /* CONFIG_BLK_CGROUP */
 373 #endif  /* _BLK_CGROUP_H */