blkcg: implement blkio_policy_type->cftypes
[deliverable/linux.git] / block / blk-cgroup.h
CommitLineData
31e4c28d
VG
1#ifndef _BLK_CGROUP_H
2#define _BLK_CGROUP_H
3/*
4 * Common Block IO controller cgroup interface
5 *
6 * Based on ideas and code from CFQ, CFS and BFQ:
7 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
8 *
9 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
10 * Paolo Valente <paolo.valente@unimore.it>
11 *
12 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
13 * Nauman Rafique <nauman@google.com>
14 */
15
16#include <linux/cgroup.h>
575969a0 17#include <linux/u64_stats_sync.h>
829fdb50 18#include <linux/seq_file.h>
31e4c28d 19
062a644d
VG
20enum blkio_policy_id {
21 BLKIO_POLICY_PROP = 0, /* Proportional Bandwidth division */
4c9eefa1 22 BLKIO_POLICY_THROTL, /* Throttling */
035d10b2
TH
23
24 BLKIO_NR_POLICIES,
062a644d
VG
25};
26
9355aede
VG
27/* Max limits for throttle policy */
28#define THROTL_IOPS_MAX UINT_MAX
29
32e380ae 30#ifdef CONFIG_BLK_CGROUP
2f5ea477 31
d3d32e69
TH
32/* cft->private [un]packing for stat printing */
33#define BLKCG_STAT_PRIV(pol, off) (((unsigned)(pol) << 16) | (off))
34#define BLKCG_STAT_POL(prv) ((unsigned)(prv) >> 16)
35#define BLKCG_STAT_OFF(prv) ((unsigned)(prv) & 0xffff)
2aa4a152 36
edcb0722
TH
37enum blkg_rwstat_type {
38 BLKG_RWSTAT_READ,
39 BLKG_RWSTAT_WRITE,
40 BLKG_RWSTAT_SYNC,
41 BLKG_RWSTAT_ASYNC,
42
43 BLKG_RWSTAT_NR,
44 BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
303a3acb
DS
45};
46
812df48d
DS
47/* blkg state flags */
48enum blkg_state_flags {
49 BLKG_waiting = 0,
50 BLKG_idling,
51 BLKG_empty,
52};
53
31e4c28d
VG
54struct blkio_cgroup {
55 struct cgroup_subsys_state css;
56 unsigned int weight;
57 spinlock_t lock;
58 struct hlist_head blkg_list;
9a9e8a26
TH
59
60 /* for policies to test whether associated blkcg has changed */
61 uint64_t id;
31e4c28d
VG
62};
63
edcb0722
TH
64struct blkg_stat {
65 struct u64_stats_sync syncp;
66 uint64_t cnt;
67};
68
69struct blkg_rwstat {
70 struct u64_stats_sync syncp;
71 uint64_t cnt[BLKG_RWSTAT_NR];
72};
73
303a3acb 74struct blkio_group_stats {
edcb0722
TH
75 /* number of ios merged */
76 struct blkg_rwstat merged;
77 /* total time spent on device in ns, may not be accurate w/ queueing */
78 struct blkg_rwstat service_time;
79 /* total time spent waiting in scheduler queue in ns */
80 struct blkg_rwstat wait_time;
81 /* number of IOs queued up */
82 struct blkg_rwstat queued;
303a3acb 83 /* total disk time and nr sectors dispatched by this group */
edcb0722 84 struct blkg_stat time;
303a3acb 85#ifdef CONFIG_DEBUG_BLK_CGROUP
edcb0722
TH
86 /* time not charged to this cgroup */
87 struct blkg_stat unaccounted_time;
88 /* sum of number of ios queued across all samples */
89 struct blkg_stat avg_queue_size_sum;
90 /* count of samples taken for average */
91 struct blkg_stat avg_queue_size_samples;
92 /* how many times this group has been removed from service tree */
93 struct blkg_stat dequeue;
94 /* total time spent waiting for it to be assigned a timeslice. */
95 struct blkg_stat group_wait_time;
96 /* time spent idling for this blkio_group */
97 struct blkg_stat idle_time;
98 /* total time with empty current active q with other requests queued */
99 struct blkg_stat empty_time;
997a026c 100 /* fields after this shouldn't be cleared on stat reset */
edcb0722
TH
101 uint64_t start_group_wait_time;
102 uint64_t start_idle_time;
103 uint64_t start_empty_time;
104 uint16_t flags;
303a3acb
DS
105#endif
106};
107
5624a4e4
VG
108/* Per cpu blkio group stats */
109struct blkio_group_stats_cpu {
edcb0722
TH
110 /* total bytes transferred */
111 struct blkg_rwstat service_bytes;
112 /* total IOs serviced, post merge */
113 struct blkg_rwstat serviced;
114 /* total sectors transferred */
115 struct blkg_stat sectors;
5624a4e4
VG
116};
117
e56da7e2
TH
118struct blkio_group_conf {
119 unsigned int weight;
c4682aec 120 u64 iops[2];
e56da7e2
TH
121 u64 bps[2];
122};
123
0381411e
TH
124/* per-blkg per-policy data */
125struct blkg_policy_data {
126 /* the blkg this per-policy data belongs to */
127 struct blkio_group *blkg;
128
549d3aa8
TH
129 /* Configuration */
130 struct blkio_group_conf conf;
131
132 struct blkio_group_stats stats;
133 /* Per cpu stats pointer */
134 struct blkio_group_stats_cpu __percpu *stats_cpu;
135
0381411e
TH
136 /* pol->pdata_size bytes of private data used by policy impl */
137 char pdata[] __aligned(__alignof__(unsigned long long));
138};
139
31e4c28d 140struct blkio_group {
c875f4d0
TH
141 /* Pointer to the associated request_queue */
142 struct request_queue *q;
e8989fae 143 struct list_head q_node;
31e4c28d 144 struct hlist_node blkcg_node;
7ee9c562 145 struct blkio_cgroup *blkcg;
2868ef7b
VG
146 /* Store cgroup path */
147 char path[128];
1adaf3dd
TH
148 /* reference count */
149 int refcnt;
22084190 150
549d3aa8 151 struct blkg_policy_data *pd[BLKIO_NR_POLICIES];
1adaf3dd 152
1cd9e039
VG
153 /* List of blkg waiting for per cpu stats memory to be allocated */
154 struct list_head alloc_node;
1adaf3dd 155 struct rcu_head rcu_head;
31e4c28d
VG
156};
157
0381411e 158typedef void (blkio_init_group_fn)(struct blkio_group *blkg);
ca32aefc 159typedef void (blkio_update_group_weight_fn)(struct request_queue *q,
fe071437 160 struct blkio_group *blkg, unsigned int weight);
ca32aefc 161typedef void (blkio_update_group_read_bps_fn)(struct request_queue *q,
fe071437 162 struct blkio_group *blkg, u64 read_bps);
ca32aefc 163typedef void (blkio_update_group_write_bps_fn)(struct request_queue *q,
fe071437 164 struct blkio_group *blkg, u64 write_bps);
ca32aefc 165typedef void (blkio_update_group_read_iops_fn)(struct request_queue *q,
fe071437 166 struct blkio_group *blkg, unsigned int read_iops);
ca32aefc 167typedef void (blkio_update_group_write_iops_fn)(struct request_queue *q,
fe071437 168 struct blkio_group *blkg, unsigned int write_iops);
3e252066
VG
169
170struct blkio_policy_ops {
0381411e 171 blkio_init_group_fn *blkio_init_group_fn;
3e252066 172 blkio_update_group_weight_fn *blkio_update_group_weight_fn;
4c9eefa1
VG
173 blkio_update_group_read_bps_fn *blkio_update_group_read_bps_fn;
174 blkio_update_group_write_bps_fn *blkio_update_group_write_bps_fn;
7702e8f4
VG
175 blkio_update_group_read_iops_fn *blkio_update_group_read_iops_fn;
176 blkio_update_group_write_iops_fn *blkio_update_group_write_iops_fn;
3e252066
VG
177};
178
179struct blkio_policy_type {
180 struct list_head list;
181 struct blkio_policy_ops ops;
062a644d 182 enum blkio_policy_id plid;
0381411e 183 size_t pdata_size; /* policy specific private data size */
44ea53de 184 struct cftype *cftypes; /* cgroup files for the policy */
3e252066
VG
185};
186
5efd6113
TH
187extern int blkcg_init_queue(struct request_queue *q);
188extern void blkcg_drain_queue(struct request_queue *q);
189extern void blkcg_exit_queue(struct request_queue *q);
190
3e252066
VG
191/* Blkio controller policy registration */
192extern void blkio_policy_register(struct blkio_policy_type *);
193extern void blkio_policy_unregister(struct blkio_policy_type *);
e8989fae
TH
194extern void blkg_destroy_all(struct request_queue *q, bool destroy_root);
195extern void update_root_blkg_pd(struct request_queue *q,
196 enum blkio_policy_id plid);
3e252066 197
829fdb50
TH
198void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg,
199 u64 (*prfill)(struct seq_file *, struct blkg_policy_data *, int),
200 int pol, int data, bool show_total);
201u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
202u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
203 const struct blkg_rwstat *rwstat);
204int blkcg_print_stat(struct cgroup *cgrp, struct cftype *cft,
205 struct seq_file *sf);
206int blkcg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
207 struct seq_file *sf);
208int blkcg_print_cpu_stat(struct cgroup *cgrp, struct cftype *cft,
209 struct seq_file *sf);
210int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
211 struct seq_file *sf);
212
213struct blkg_conf_ctx {
214 struct gendisk *disk;
215 struct blkio_group *blkg;
216 u64 v;
217};
218
219int blkg_conf_prep(struct blkio_cgroup *blkcg, const char *input,
220 struct blkg_conf_ctx *ctx);
221void blkg_conf_finish(struct blkg_conf_ctx *ctx);
222
223
0381411e
TH
224/**
225 * blkg_to_pdata - get policy private data
226 * @blkg: blkg of interest
227 * @pol: policy of interest
228 *
229 * Return pointer to private data associated with the @blkg-@pol pair.
230 */
231static inline void *blkg_to_pdata(struct blkio_group *blkg,
232 struct blkio_policy_type *pol)
233{
549d3aa8 234 return blkg ? blkg->pd[pol->plid]->pdata : NULL;
0381411e
TH
235}
236
237/**
238 * pdata_to_blkg - get blkg associated with policy private data
239 * @pdata: policy private data of interest
0381411e 240 *
aaec55a0 241 * @pdata is policy private data. Determine the blkg it's associated with.
0381411e 242 */
aaec55a0 243static inline struct blkio_group *pdata_to_blkg(void *pdata)
0381411e
TH
244{
245 if (pdata) {
246 struct blkg_policy_data *pd =
247 container_of(pdata, struct blkg_policy_data, pdata);
248 return pd->blkg;
249 }
250 return NULL;
251}
252
afc24d49
VG
253static inline char *blkg_path(struct blkio_group *blkg)
254{
255 return blkg->path;
256}
257
1adaf3dd
TH
258/**
259 * blkg_get - get a blkg reference
260 * @blkg: blkg to get
261 *
262 * The caller should be holding queue_lock and an existing reference.
263 */
264static inline void blkg_get(struct blkio_group *blkg)
265{
266 lockdep_assert_held(blkg->q->queue_lock);
267 WARN_ON_ONCE(!blkg->refcnt);
268 blkg->refcnt++;
269}
270
271void __blkg_release(struct blkio_group *blkg);
272
273/**
274 * blkg_put - put a blkg reference
275 * @blkg: blkg to put
276 *
277 * The caller should be holding queue_lock.
278 */
279static inline void blkg_put(struct blkio_group *blkg)
280{
281 lockdep_assert_held(blkg->q->queue_lock);
282 WARN_ON_ONCE(blkg->refcnt <= 0);
283 if (!--blkg->refcnt)
284 __blkg_release(blkg);
285}
286
edcb0722
TH
287/**
288 * blkg_stat_add - add a value to a blkg_stat
289 * @stat: target blkg_stat
290 * @val: value to add
291 *
292 * Add @val to @stat. The caller is responsible for synchronizing calls to
293 * this function.
294 */
295static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
296{
297 u64_stats_update_begin(&stat->syncp);
298 stat->cnt += val;
299 u64_stats_update_end(&stat->syncp);
300}
301
302/**
303 * blkg_stat_read - read the current value of a blkg_stat
304 * @stat: blkg_stat to read
305 *
306 * Read the current value of @stat. This function can be called without
307 * synchroniztion and takes care of u64 atomicity.
308 */
309static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
310{
311 unsigned int start;
312 uint64_t v;
313
314 do {
315 start = u64_stats_fetch_begin(&stat->syncp);
316 v = stat->cnt;
317 } while (u64_stats_fetch_retry(&stat->syncp, start));
318
319 return v;
320}
321
322/**
323 * blkg_stat_reset - reset a blkg_stat
324 * @stat: blkg_stat to reset
325 */
326static inline void blkg_stat_reset(struct blkg_stat *stat)
327{
328 stat->cnt = 0;
329}
330
331/**
332 * blkg_rwstat_add - add a value to a blkg_rwstat
333 * @rwstat: target blkg_rwstat
334 * @rw: mask of REQ_{WRITE|SYNC}
335 * @val: value to add
336 *
337 * Add @val to @rwstat. The counters are chosen according to @rw. The
338 * caller is responsible for synchronizing calls to this function.
339 */
340static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
341 int rw, uint64_t val)
342{
343 u64_stats_update_begin(&rwstat->syncp);
344
345 if (rw & REQ_WRITE)
346 rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
347 else
348 rwstat->cnt[BLKG_RWSTAT_READ] += val;
349 if (rw & REQ_SYNC)
350 rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
351 else
352 rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;
353
354 u64_stats_update_end(&rwstat->syncp);
355}
356
357/**
358 * blkg_rwstat_read - read the current values of a blkg_rwstat
359 * @rwstat: blkg_rwstat to read
360 *
361 * Read the current snapshot of @rwstat and return it as the return value.
362 * This function can be called without synchronization and takes care of
363 * u64 atomicity.
364 */
365static struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
366{
367 unsigned int start;
368 struct blkg_rwstat tmp;
369
370 do {
371 start = u64_stats_fetch_begin(&rwstat->syncp);
372 tmp = *rwstat;
373 } while (u64_stats_fetch_retry(&rwstat->syncp, start));
374
375 return tmp;
376}
377
378/**
379 * blkg_rwstat_sum - read the total count of a blkg_rwstat
380 * @rwstat: blkg_rwstat to read
381 *
382 * Return the total count of @rwstat regardless of the IO direction. This
383 * function can be called without synchronization and takes care of u64
384 * atomicity.
385 */
386static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat)
387{
388 struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
389
390 return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
391}
392
393/**
394 * blkg_rwstat_reset - reset a blkg_rwstat
395 * @rwstat: blkg_rwstat to reset
396 */
397static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
398{
399 memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
400}
401
2f5ea477
JA
402#else
403
404struct blkio_group {
405};
406
3e252066
VG
407struct blkio_policy_type {
408};
409
5efd6113
TH
410static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
411static inline void blkcg_drain_queue(struct request_queue *q) { }
412static inline void blkcg_exit_queue(struct request_queue *q) { }
3e252066
VG
413static inline void blkio_policy_register(struct blkio_policy_type *blkiop) { }
414static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { }
03aa264a 415static inline void blkg_destroy_all(struct request_queue *q,
03aa264a 416 bool destory_root) { }
e8989fae
TH
417static inline void update_root_blkg_pd(struct request_queue *q,
418 enum blkio_policy_id plid) { }
3e252066 419
0381411e
TH
420static inline void *blkg_to_pdata(struct blkio_group *blkg,
421 struct blkio_policy_type *pol) { return NULL; }
422static inline struct blkio_group *pdata_to_blkg(void *pdata,
423 struct blkio_policy_type *pol) { return NULL; }
afc24d49 424static inline char *blkg_path(struct blkio_group *blkg) { return NULL; }
1adaf3dd
TH
425static inline void blkg_get(struct blkio_group *blkg) { }
426static inline void blkg_put(struct blkio_group *blkg) { }
afc24d49 427
2f5ea477
JA
428#endif
429
df457f84 430#define BLKIO_WEIGHT_MIN 10
31e4c28d
VG
431#define BLKIO_WEIGHT_MAX 1000
432#define BLKIO_WEIGHT_DEFAULT 500
433
2868ef7b 434#ifdef CONFIG_DEBUG_BLK_CGROUP
c1768268
TH
435void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
436 struct blkio_policy_type *pol);
9195291e 437void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
c1768268
TH
438 struct blkio_policy_type *pol,
439 unsigned long dequeue);
440void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
441 struct blkio_policy_type *pol);
442void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
443 struct blkio_policy_type *pol);
444void blkiocg_set_start_empty_time(struct blkio_group *blkg,
445 struct blkio_policy_type *pol);
812df48d
DS
446
447#define BLKG_FLAG_FNS(name) \
448static inline void blkio_mark_blkg_##name( \
449 struct blkio_group_stats *stats) \
450{ \
451 stats->flags |= (1 << BLKG_##name); \
452} \
453static inline void blkio_clear_blkg_##name( \
454 struct blkio_group_stats *stats) \
455{ \
456 stats->flags &= ~(1 << BLKG_##name); \
457} \
458static inline int blkio_blkg_##name(struct blkio_group_stats *stats) \
459{ \
460 return (stats->flags & (1 << BLKG_##name)) != 0; \
461} \
462
463BLKG_FLAG_FNS(waiting)
464BLKG_FLAG_FNS(idling)
465BLKG_FLAG_FNS(empty)
466#undef BLKG_FLAG_FNS
2868ef7b 467#else
c1768268
TH
468static inline void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
469 struct blkio_policy_type *pol) { }
9195291e 470static inline void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
c1768268
TH
471 struct blkio_policy_type *pol, unsigned long dequeue) { }
472static inline void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
473 struct blkio_policy_type *pol) { }
474static inline void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
475 struct blkio_policy_type *pol) { }
476static inline void blkiocg_set_start_empty_time(struct blkio_group *blkg,
477 struct blkio_policy_type *pol) { }
2868ef7b
VG
478#endif
479
32e380ae 480#ifdef CONFIG_BLK_CGROUP
31e4c28d
VG
481extern struct blkio_cgroup blkio_root_cgroup;
482extern struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup);
4f85cb96 483extern struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio);
cd1604fa 484extern struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
e8989fae 485 struct request_queue *q);
cd1604fa
TH
486struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
487 struct request_queue *q,
cd1604fa 488 bool for_root);
303a3acb 489void blkiocg_update_timeslice_used(struct blkio_group *blkg,
c1768268
TH
490 struct blkio_policy_type *pol,
491 unsigned long time,
492 unsigned long unaccounted_time);
493void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
494 struct blkio_policy_type *pol,
495 uint64_t bytes, bool direction, bool sync);
84c124da 496void blkiocg_update_completion_stats(struct blkio_group *blkg,
c1768268
TH
497 struct blkio_policy_type *pol,
498 uint64_t start_time,
499 uint64_t io_start_time, bool direction,
500 bool sync);
501void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
502 struct blkio_policy_type *pol,
503 bool direction, bool sync);
a11cdaa7 504void blkiocg_update_io_add_stats(struct blkio_group *blkg,
c1768268
TH
505 struct blkio_policy_type *pol,
506 struct blkio_group *curr_blkg, bool direction,
507 bool sync);
a11cdaa7 508void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
c1768268
TH
509 struct blkio_policy_type *pol,
510 bool direction, bool sync);
31e4c28d 511#else
2f5ea477 512struct cgroup;
31e4c28d
VG
513static inline struct blkio_cgroup *
514cgroup_to_blkio_cgroup(struct cgroup *cgroup) { return NULL; }
70087dc3 515static inline struct blkio_cgroup *
4f85cb96 516bio_blkio_cgroup(struct bio *bio) { return NULL; }
31e4c28d 517
cd1604fa
TH
518static inline struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
519 void *key) { return NULL; }
303a3acb 520static inline void blkiocg_update_timeslice_used(struct blkio_group *blkg,
c1768268
TH
521 struct blkio_policy_type *pol, unsigned long time,
522 unsigned long unaccounted_time) { }
84c124da 523static inline void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
c1768268
TH
524 struct blkio_policy_type *pol, uint64_t bytes,
525 bool direction, bool sync) { }
84c124da 526static inline void blkiocg_update_completion_stats(struct blkio_group *blkg,
c1768268
TH
527 struct blkio_policy_type *pol, uint64_t start_time,
528 uint64_t io_start_time, bool direction, bool sync) { }
812d4026 529static inline void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
c1768268
TH
530 struct blkio_policy_type *pol, bool direction,
531 bool sync) { }
a11cdaa7 532static inline void blkiocg_update_io_add_stats(struct blkio_group *blkg,
c1768268
TH
533 struct blkio_policy_type *pol,
534 struct blkio_group *curr_blkg, bool direction,
535 bool sync) { }
a11cdaa7 536static inline void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
c1768268
TH
537 struct blkio_policy_type *pol, bool direction,
538 bool sync) { }
31e4c28d
VG
539#endif
540#endif /* _BLK_CGROUP_H */
This page took 0.146342 seconds and 5 git commands to generate.