2 * Copyright (C) 2009-2011 Red Hat, Inc.
4 * Author: Mikulas Patocka <mpatocka@redhat.com>
6 * This file is released under the GPL.
11 #include <linux/device-mapper.h>
12 #include <linux/dm-io.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
15 #include <linux/shrinker.h>
16 #include <linux/module.h>
18 #define DM_MSG_PREFIX "bufio"
21 * Memory management policy:
22 * Limit the number of buffers to DM_BUFIO_MEMORY_PERCENT of main memory
23 * or DM_BUFIO_VMALLOC_PERCENT of vmalloc memory (whichever is lower).
24 * Always allocate at least DM_BUFIO_MIN_BUFFERS buffers.
25 * Start background writeback when there are DM_BUFIO_WRITEBACK_PERCENT
28 #define DM_BUFIO_MIN_BUFFERS 8
30 #define DM_BUFIO_MEMORY_PERCENT 2
31 #define DM_BUFIO_VMALLOC_PERCENT 25
32 #define DM_BUFIO_WRITEBACK_PERCENT 75
35 * Check buffer ages in this interval (seconds)
37 #define DM_BUFIO_WORK_TIMER_SECS 10
40 * Free buffers when they are older than this (seconds)
42 #define DM_BUFIO_DEFAULT_AGE_SECS 60
45 * The number of bvec entries that are embedded directly in the buffer.
46 * If the chunk size is larger, dm-io is used to do the io.
48 #define DM_BUFIO_INLINE_VECS 16
53 #define DM_BUFIO_HASH_BITS 20
54 #define DM_BUFIO_HASH(block) \
55 ((((block) >> DM_BUFIO_HASH_BITS) ^ (block)) & \
56 ((1 << DM_BUFIO_HASH_BITS) - 1))
59 * Don't try to use kmem_cache_alloc for blocks larger than this.
60 * For explanation, see alloc_buffer_data below.
62 #define DM_BUFIO_BLOCK_SIZE_SLAB_LIMIT (PAGE_SIZE >> 1)
63 #define DM_BUFIO_BLOCK_SIZE_GFP_LIMIT (PAGE_SIZE << (MAX_ORDER - 1))
66 * dm_buffer->list_mode
74 * All buffers are linked to cache_hash with their hash_list field.
76 * Clean buffers that are not being written (B_WRITING not set)
77 * are linked to lru[LIST_CLEAN] with their lru_list field.
79 * Dirty and clean buffers that are being written are linked to
80 * lru[LIST_DIRTY] with their lru_list field. When the write
81 * finishes, the buffer cannot be relinked immediately (because we
82 * are in an interrupt context and relinking requires process
83 * context), so some clean-not-writing buffers can be held on
84 * dirty_lru too. They are later added to lru in the process
87 struct dm_bufio_client
{
90 struct list_head lru
[LIST_SIZE
];
91 unsigned long n_buffers
[LIST_SIZE
];
93 struct block_device
*bdev
;
95 unsigned char sectors_per_block_bits
;
96 unsigned char pages_per_block_bits
;
97 unsigned char blocks_per_page_bits
;
99 void (*alloc_callback
)(struct dm_buffer
*);
100 void (*write_callback
)(struct dm_buffer
*);
102 struct dm_io_client
*dm_io
;
104 struct list_head reserved_buffers
;
105 unsigned need_reserved_buffers
;
107 struct hlist_head
*cache_hash
;
108 wait_queue_head_t free_buffer_wait
;
110 int async_write_error
;
112 struct list_head client_list
;
113 struct shrinker shrinker
;
124 * Describes how the block was allocated:
125 * kmem_cache_alloc(), __get_free_pages() or vmalloc().
126 * See the comment at alloc_buffer_data.
130 DATA_MODE_GET_FREE_PAGES
= 1,
131 DATA_MODE_VMALLOC
= 2,
136 struct hlist_node hash_list
;
137 struct list_head lru_list
;
140 enum data_mode data_mode
;
141 unsigned char list_mode
; /* LIST_* */
146 unsigned long last_accessed
;
147 struct dm_bufio_client
*c
;
149 struct bio_vec bio_vec
[DM_BUFIO_INLINE_VECS
];
152 /*----------------------------------------------------------------*/
154 static struct kmem_cache
*dm_bufio_caches
[PAGE_SHIFT
- SECTOR_SHIFT
];
155 static char *dm_bufio_cache_names
[PAGE_SHIFT
- SECTOR_SHIFT
];
157 static inline int dm_bufio_cache_index(struct dm_bufio_client
*c
)
159 unsigned ret
= c
->blocks_per_page_bits
- 1;
161 BUG_ON(ret
>= ARRAY_SIZE(dm_bufio_caches
));
166 #define DM_BUFIO_CACHE(c) (dm_bufio_caches[dm_bufio_cache_index(c)])
167 #define DM_BUFIO_CACHE_NAME(c) (dm_bufio_cache_names[dm_bufio_cache_index(c)])
169 #define dm_bufio_in_request() (!!current->bio_list)
171 static void dm_bufio_lock(struct dm_bufio_client
*c
)
173 mutex_lock_nested(&c
->lock
, dm_bufio_in_request());
176 static int dm_bufio_trylock(struct dm_bufio_client
*c
)
178 return mutex_trylock(&c
->lock
);
181 static void dm_bufio_unlock(struct dm_bufio_client
*c
)
183 mutex_unlock(&c
->lock
);
187 * FIXME Move to sched.h?
189 #ifdef CONFIG_PREEMPT_VOLUNTARY
190 # define dm_bufio_cond_resched() \
192 if (unlikely(need_resched())) \
196 # define dm_bufio_cond_resched() do { } while (0)
199 /*----------------------------------------------------------------*/
202 * Default cache size: available memory divided by the ratio.
204 static unsigned long dm_bufio_default_cache_size
;
207 * Total cache size set by the user.
209 static unsigned long dm_bufio_cache_size
;
212 * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change
213 * at any time. If it disagrees, the user has changed cache size.
215 static unsigned long dm_bufio_cache_size_latch
;
217 static DEFINE_SPINLOCK(param_spinlock
);
220 * Buffers are freed after this timeout
222 static unsigned dm_bufio_max_age
= DM_BUFIO_DEFAULT_AGE_SECS
;
224 static unsigned long dm_bufio_peak_allocated
;
225 static unsigned long dm_bufio_allocated_kmem_cache
;
226 static unsigned long dm_bufio_allocated_get_free_pages
;
227 static unsigned long dm_bufio_allocated_vmalloc
;
228 static unsigned long dm_bufio_current_allocated
;
230 /*----------------------------------------------------------------*/
233 * Per-client cache: dm_bufio_cache_size / dm_bufio_client_count
235 static unsigned long dm_bufio_cache_size_per_client
;
238 * The current number of clients.
240 static int dm_bufio_client_count
;
243 * The list of all clients.
245 static LIST_HEAD(dm_bufio_all_clients
);
248 * This mutex protects dm_bufio_cache_size_latch,
249 * dm_bufio_cache_size_per_client and dm_bufio_client_count
251 static DEFINE_MUTEX(dm_bufio_clients_lock
);
253 /*----------------------------------------------------------------*/
255 static void adjust_total_allocated(enum data_mode data_mode
, long diff
)
257 static unsigned long * const class_ptr
[DATA_MODE_LIMIT
] = {
258 &dm_bufio_allocated_kmem_cache
,
259 &dm_bufio_allocated_get_free_pages
,
260 &dm_bufio_allocated_vmalloc
,
263 spin_lock(¶m_spinlock
);
265 *class_ptr
[data_mode
] += diff
;
267 dm_bufio_current_allocated
+= diff
;
269 if (dm_bufio_current_allocated
> dm_bufio_peak_allocated
)
270 dm_bufio_peak_allocated
= dm_bufio_current_allocated
;
272 spin_unlock(¶m_spinlock
);
276 * Change the number of clients and recalculate per-client limit.
278 static void __cache_size_refresh(void)
280 BUG_ON(!mutex_is_locked(&dm_bufio_clients_lock
));
281 BUG_ON(dm_bufio_client_count
< 0);
283 dm_bufio_cache_size_latch
= dm_bufio_cache_size
;
288 * Use default if set to 0 and report the actual cache size used.
290 if (!dm_bufio_cache_size_latch
) {
291 (void)cmpxchg(&dm_bufio_cache_size
, 0,
292 dm_bufio_default_cache_size
);
293 dm_bufio_cache_size_latch
= dm_bufio_default_cache_size
;
296 dm_bufio_cache_size_per_client
= dm_bufio_cache_size_latch
/
297 (dm_bufio_client_count
? : 1);
301 * Allocating buffer data.
303 * Small buffers are allocated with kmem_cache, to use space optimally.
305 * For large buffers, we choose between get_free_pages and vmalloc.
306 * Each has advantages and disadvantages.
308 * __get_free_pages can randomly fail if the memory is fragmented.
309 * __vmalloc won't randomly fail, but vmalloc space is limited (it may be
310 * as low as 128M) so using it for caching is not appropriate.
312 * If the allocation may fail we use __get_free_pages. Memory fragmentation
313 * won't have a fatal effect here, but it just causes flushes of some other
314 * buffers and more I/O will be performed. Don't use __get_free_pages if it
315 * always fails (i.e. order >= MAX_ORDER).
317 * If the allocation shouldn't fail we use __vmalloc. This is only for the
318 * initial reserve allocation, so there's no risk of wasting all vmalloc
321 static void *alloc_buffer_data(struct dm_bufio_client
*c
, gfp_t gfp_mask
,
322 enum data_mode
*data_mode
)
324 if (c
->block_size
<= DM_BUFIO_BLOCK_SIZE_SLAB_LIMIT
) {
325 *data_mode
= DATA_MODE_SLAB
;
326 return kmem_cache_alloc(DM_BUFIO_CACHE(c
), gfp_mask
);
329 if (c
->block_size
<= DM_BUFIO_BLOCK_SIZE_GFP_LIMIT
&&
330 gfp_mask
& __GFP_NORETRY
) {
331 *data_mode
= DATA_MODE_GET_FREE_PAGES
;
332 return (void *)__get_free_pages(gfp_mask
,
333 c
->pages_per_block_bits
);
336 *data_mode
= DATA_MODE_VMALLOC
;
337 return __vmalloc(c
->block_size
, gfp_mask
, PAGE_KERNEL
);
341 * Free buffer's data.
343 static void free_buffer_data(struct dm_bufio_client
*c
,
344 void *data
, enum data_mode data_mode
)
348 kmem_cache_free(DM_BUFIO_CACHE(c
), data
);
351 case DATA_MODE_GET_FREE_PAGES
:
352 free_pages((unsigned long)data
, c
->pages_per_block_bits
);
355 case DATA_MODE_VMALLOC
:
360 DMCRIT("dm_bufio_free_buffer_data: bad data mode: %d",
367 * Allocate buffer and its data.
369 static struct dm_buffer
*alloc_buffer(struct dm_bufio_client
*c
, gfp_t gfp_mask
)
371 struct dm_buffer
*b
= kmalloc(sizeof(struct dm_buffer
) + c
->aux_size
,
379 b
->data
= alloc_buffer_data(c
, gfp_mask
, &b
->data_mode
);
385 adjust_total_allocated(b
->data_mode
, (long)c
->block_size
);
391 * Free buffer and its data.
393 static void free_buffer(struct dm_buffer
*b
)
395 struct dm_bufio_client
*c
= b
->c
;
397 adjust_total_allocated(b
->data_mode
, -(long)c
->block_size
);
399 free_buffer_data(c
, b
->data
, b
->data_mode
);
404 * Link buffer to the hash list and clean or dirty queue.
406 static void __link_buffer(struct dm_buffer
*b
, sector_t block
, int dirty
)
408 struct dm_bufio_client
*c
= b
->c
;
410 c
->n_buffers
[dirty
]++;
412 b
->list_mode
= dirty
;
413 list_add(&b
->lru_list
, &c
->lru
[dirty
]);
414 hlist_add_head(&b
->hash_list
, &c
->cache_hash
[DM_BUFIO_HASH(block
)]);
415 b
->last_accessed
= jiffies
;
419 * Unlink buffer from the hash list and dirty or clean queue.
421 static void __unlink_buffer(struct dm_buffer
*b
)
423 struct dm_bufio_client
*c
= b
->c
;
425 BUG_ON(!c
->n_buffers
[b
->list_mode
]);
427 c
->n_buffers
[b
->list_mode
]--;
428 hlist_del(&b
->hash_list
);
429 list_del(&b
->lru_list
);
433 * Place the buffer to the head of dirty or clean LRU queue.
435 static void __relink_lru(struct dm_buffer
*b
, int dirty
)
437 struct dm_bufio_client
*c
= b
->c
;
439 BUG_ON(!c
->n_buffers
[b
->list_mode
]);
441 c
->n_buffers
[b
->list_mode
]--;
442 c
->n_buffers
[dirty
]++;
443 b
->list_mode
= dirty
;
444 list_del(&b
->lru_list
);
445 list_add(&b
->lru_list
, &c
->lru
[dirty
]);
448 /*----------------------------------------------------------------
449 * Submit I/O on the buffer.
451 * Bio interface is faster but it has some problems:
452 * the vector list is limited (increasing this limit increases
453 * memory-consumption per buffer, so it is not viable);
455 * the memory must be direct-mapped, not vmalloced;
457 * the I/O driver can reject requests spuriously if it thinks that
458 * the requests are too big for the device or if they cross a
459 * controller-defined memory boundary.
461 * If the buffer is small enough (up to DM_BUFIO_INLINE_VECS pages) and
462 * it is not vmalloced, try using the bio interface.
464 * If the buffer is big, if it is vmalloced or if the underlying device
465 * rejects the bio because it is too large, use dm-io layer to do the I/O.
466 * The dm-io layer splits the I/O into multiple requests, avoiding the above
468 *--------------------------------------------------------------*/
471 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
472 * that the request was handled directly with bio interface.
474 static void dmio_complete(unsigned long error
, void *context
)
476 struct dm_buffer
*b
= context
;
478 b
->bio
.bi_end_io(&b
->bio
, error
? -EIO
: 0);
481 static void use_dmio(struct dm_buffer
*b
, int rw
, sector_t block
,
482 bio_end_io_t
*end_io
)
485 struct dm_io_request io_req
= {
487 .notify
.fn
= dmio_complete
,
489 .client
= b
->c
->dm_io
,
491 struct dm_io_region region
= {
493 .sector
= block
<< b
->c
->sectors_per_block_bits
,
494 .count
= b
->c
->block_size
>> SECTOR_SHIFT
,
497 if (b
->data_mode
!= DATA_MODE_VMALLOC
) {
498 io_req
.mem
.type
= DM_IO_KMEM
;
499 io_req
.mem
.ptr
.addr
= b
->data
;
501 io_req
.mem
.type
= DM_IO_VMA
;
502 io_req
.mem
.ptr
.vma
= b
->data
;
505 b
->bio
.bi_end_io
= end_io
;
507 r
= dm_io(&io_req
, 1, ®ion
, NULL
);
512 static void use_inline_bio(struct dm_buffer
*b
, int rw
, sector_t block
,
513 bio_end_io_t
*end_io
)
519 b
->bio
.bi_io_vec
= b
->bio_vec
;
520 b
->bio
.bi_max_vecs
= DM_BUFIO_INLINE_VECS
;
521 b
->bio
.bi_sector
= block
<< b
->c
->sectors_per_block_bits
;
522 b
->bio
.bi_bdev
= b
->c
->bdev
;
523 b
->bio
.bi_end_io
= end_io
;
526 * We assume that if len >= PAGE_SIZE ptr is page-aligned.
527 * If len < PAGE_SIZE the buffer doesn't cross page boundary.
530 len
= b
->c
->block_size
;
532 if (len
>= PAGE_SIZE
)
533 BUG_ON((unsigned long)ptr
& (PAGE_SIZE
- 1));
535 BUG_ON((unsigned long)ptr
& (len
- 1));
538 if (!bio_add_page(&b
->bio
, virt_to_page(ptr
),
539 len
< PAGE_SIZE
? len
: PAGE_SIZE
,
540 virt_to_phys(ptr
) & (PAGE_SIZE
- 1))) {
541 BUG_ON(b
->c
->block_size
<= PAGE_SIZE
);
542 use_dmio(b
, rw
, block
, end_io
);
550 submit_bio(rw
, &b
->bio
);
553 static void submit_io(struct dm_buffer
*b
, int rw
, sector_t block
,
554 bio_end_io_t
*end_io
)
556 if (rw
== WRITE
&& b
->c
->write_callback
)
557 b
->c
->write_callback(b
);
559 if (b
->c
->block_size
<= DM_BUFIO_INLINE_VECS
* PAGE_SIZE
&&
560 b
->data_mode
!= DATA_MODE_VMALLOC
)
561 use_inline_bio(b
, rw
, block
, end_io
);
563 use_dmio(b
, rw
, block
, end_io
);
566 /*----------------------------------------------------------------
567 * Writing dirty buffers
568 *--------------------------------------------------------------*/
571 * The endio routine for write.
573 * Set the error, clear B_WRITING bit and wake anyone who was waiting on
576 static void write_endio(struct bio
*bio
, int error
)
578 struct dm_buffer
*b
= container_of(bio
, struct dm_buffer
, bio
);
580 b
->write_error
= error
;
582 struct dm_bufio_client
*c
= b
->c
;
583 (void)cmpxchg(&c
->async_write_error
, 0, error
);
586 BUG_ON(!test_bit(B_WRITING
, &b
->state
));
588 smp_mb__before_clear_bit();
589 clear_bit(B_WRITING
, &b
->state
);
590 smp_mb__after_clear_bit();
592 wake_up_bit(&b
->state
, B_WRITING
);
596 * This function is called when wait_on_bit is actually waiting.
598 static int do_io_schedule(void *word
)
606 * Initiate a write on a dirty buffer, but don't wait for it.
608 * - If the buffer is not dirty, exit.
609 * - If there some previous write going on, wait for it to finish (we can't
610 * have two writes on the same buffer simultaneously).
611 * - Submit our write and don't wait on it. We set B_WRITING indicating
612 * that there is a write in progress.
614 static void __write_dirty_buffer(struct dm_buffer
*b
)
616 if (!test_bit(B_DIRTY
, &b
->state
))
619 clear_bit(B_DIRTY
, &b
->state
);
620 wait_on_bit_lock(&b
->state
, B_WRITING
,
621 do_io_schedule
, TASK_UNINTERRUPTIBLE
);
623 submit_io(b
, WRITE
, b
->block
, write_endio
);
627 * Wait until any activity on the buffer finishes. Possibly write the
628 * buffer if it is dirty. When this function finishes, there is no I/O
629 * running on the buffer and the buffer is not dirty.
631 static void __make_buffer_clean(struct dm_buffer
*b
)
633 BUG_ON(b
->hold_count
);
635 if (!b
->state
) /* fast case */
638 wait_on_bit(&b
->state
, B_READING
, do_io_schedule
, TASK_UNINTERRUPTIBLE
);
639 __write_dirty_buffer(b
);
640 wait_on_bit(&b
->state
, B_WRITING
, do_io_schedule
, TASK_UNINTERRUPTIBLE
);
644 * Find some buffer that is not held by anybody, clean it, unlink it and
647 static struct dm_buffer
*__get_unclaimed_buffer(struct dm_bufio_client
*c
)
651 list_for_each_entry_reverse(b
, &c
->lru
[LIST_CLEAN
], lru_list
) {
652 BUG_ON(test_bit(B_WRITING
, &b
->state
));
653 BUG_ON(test_bit(B_DIRTY
, &b
->state
));
655 if (!b
->hold_count
) {
656 __make_buffer_clean(b
);
660 dm_bufio_cond_resched();
663 list_for_each_entry_reverse(b
, &c
->lru
[LIST_DIRTY
], lru_list
) {
664 BUG_ON(test_bit(B_READING
, &b
->state
));
666 if (!b
->hold_count
) {
667 __make_buffer_clean(b
);
671 dm_bufio_cond_resched();
678 * Wait until some other threads free some buffer or release hold count on
681 * This function is entered with c->lock held, drops it and regains it
684 static void __wait_for_free_buffer(struct dm_bufio_client
*c
)
686 DECLARE_WAITQUEUE(wait
, current
);
688 add_wait_queue(&c
->free_buffer_wait
, &wait
);
689 set_task_state(current
, TASK_UNINTERRUPTIBLE
);
694 set_task_state(current
, TASK_RUNNING
);
695 remove_wait_queue(&c
->free_buffer_wait
, &wait
);
701 * Allocate a new buffer. If the allocation is not possible, wait until
702 * some other thread frees a buffer.
704 * May drop the lock and regain it.
706 static struct dm_buffer
*__alloc_buffer_wait_no_callback(struct dm_bufio_client
*c
)
711 * dm-bufio is resistant to allocation failures (it just keeps
712 * one buffer reserved in cases all the allocations fail).
713 * So set flags to not try too hard:
714 * GFP_NOIO: don't recurse into the I/O layer
715 * __GFP_NORETRY: don't retry and rather return failure
716 * __GFP_NOMEMALLOC: don't use emergency reserves
717 * __GFP_NOWARN: don't print a warning in case of failure
719 * For debugging, if we set the cache size to 1, no new buffers will
723 if (dm_bufio_cache_size_latch
!= 1) {
724 b
= alloc_buffer(c
, GFP_NOIO
| __GFP_NORETRY
| __GFP_NOMEMALLOC
| __GFP_NOWARN
);
729 if (!list_empty(&c
->reserved_buffers
)) {
730 b
= list_entry(c
->reserved_buffers
.next
,
731 struct dm_buffer
, lru_list
);
732 list_del(&b
->lru_list
);
733 c
->need_reserved_buffers
++;
738 b
= __get_unclaimed_buffer(c
);
742 __wait_for_free_buffer(c
);
746 static struct dm_buffer
*__alloc_buffer_wait(struct dm_bufio_client
*c
)
748 struct dm_buffer
*b
= __alloc_buffer_wait_no_callback(c
);
750 if (c
->alloc_callback
)
751 c
->alloc_callback(b
);
757 * Free a buffer and wake other threads waiting for free buffers.
759 static void __free_buffer_wake(struct dm_buffer
*b
)
761 struct dm_bufio_client
*c
= b
->c
;
763 if (!c
->need_reserved_buffers
)
766 list_add(&b
->lru_list
, &c
->reserved_buffers
);
767 c
->need_reserved_buffers
--;
770 wake_up(&c
->free_buffer_wait
);
773 static void __write_dirty_buffers_async(struct dm_bufio_client
*c
, int no_wait
)
775 struct dm_buffer
*b
, *tmp
;
777 list_for_each_entry_safe_reverse(b
, tmp
, &c
->lru
[LIST_DIRTY
], lru_list
) {
778 BUG_ON(test_bit(B_READING
, &b
->state
));
780 if (!test_bit(B_DIRTY
, &b
->state
) &&
781 !test_bit(B_WRITING
, &b
->state
)) {
782 __relink_lru(b
, LIST_CLEAN
);
786 if (no_wait
&& test_bit(B_WRITING
, &b
->state
))
789 __write_dirty_buffer(b
);
790 dm_bufio_cond_resched();
795 * Get writeback threshold and buffer limit for a given client.
797 static void __get_memory_limit(struct dm_bufio_client
*c
,
798 unsigned long *threshold_buffers
,
799 unsigned long *limit_buffers
)
801 unsigned long buffers
;
803 if (dm_bufio_cache_size
!= dm_bufio_cache_size_latch
) {
804 mutex_lock(&dm_bufio_clients_lock
);
805 __cache_size_refresh();
806 mutex_unlock(&dm_bufio_clients_lock
);
809 buffers
= dm_bufio_cache_size_per_client
>>
810 (c
->sectors_per_block_bits
+ SECTOR_SHIFT
);
812 if (buffers
< DM_BUFIO_MIN_BUFFERS
)
813 buffers
= DM_BUFIO_MIN_BUFFERS
;
815 *limit_buffers
= buffers
;
816 *threshold_buffers
= buffers
* DM_BUFIO_WRITEBACK_PERCENT
/ 100;
820 * Check if we're over watermark.
821 * If we are over threshold_buffers, start freeing buffers.
822 * If we're over "limit_buffers", block until we get under the limit.
824 static void __check_watermark(struct dm_bufio_client
*c
)
826 unsigned long threshold_buffers
, limit_buffers
;
828 __get_memory_limit(c
, &threshold_buffers
, &limit_buffers
);
830 while (c
->n_buffers
[LIST_CLEAN
] + c
->n_buffers
[LIST_DIRTY
] >
833 struct dm_buffer
*b
= __get_unclaimed_buffer(c
);
838 __free_buffer_wake(b
);
839 dm_bufio_cond_resched();
842 if (c
->n_buffers
[LIST_DIRTY
] > threshold_buffers
)
843 __write_dirty_buffers_async(c
, 1);
847 * Find a buffer in the hash.
849 static struct dm_buffer
*__find(struct dm_bufio_client
*c
, sector_t block
)
852 struct hlist_node
*hn
;
854 hlist_for_each_entry(b
, hn
, &c
->cache_hash
[DM_BUFIO_HASH(block
)],
856 dm_bufio_cond_resched();
857 if (b
->block
== block
)
864 /*----------------------------------------------------------------
866 *--------------------------------------------------------------*/
874 static struct dm_buffer
*__bufio_new(struct dm_bufio_client
*c
, sector_t block
,
875 enum new_flag nf
, struct dm_buffer
**bp
,
878 struct dm_buffer
*b
, *new_b
= NULL
;
882 b
= __find(c
, block
);
885 __relink_lru(b
, test_bit(B_DIRTY
, &b
->state
) ||
886 test_bit(B_WRITING
, &b
->state
));
893 new_b
= __alloc_buffer_wait(c
);
896 * We've had a period where the mutex was unlocked, so need to
897 * recheck the hash table.
899 b
= __find(c
, block
);
901 __free_buffer_wake(new_b
);
903 __relink_lru(b
, test_bit(B_DIRTY
, &b
->state
) ||
904 test_bit(B_WRITING
, &b
->state
));
908 __check_watermark(c
);
914 __link_buffer(b
, block
, LIST_CLEAN
);
916 if (nf
== NF_FRESH
) {
921 b
->state
= 1 << B_READING
;
928 * The endio routine for reading: set the error, clear the bit and wake up
929 * anyone waiting on the buffer.
931 static void read_endio(struct bio
*bio
, int error
)
933 struct dm_buffer
*b
= container_of(bio
, struct dm_buffer
, bio
);
935 b
->read_error
= error
;
937 BUG_ON(!test_bit(B_READING
, &b
->state
));
939 smp_mb__before_clear_bit();
940 clear_bit(B_READING
, &b
->state
);
941 smp_mb__after_clear_bit();
943 wake_up_bit(&b
->state
, B_READING
);
947 * A common routine for dm_bufio_new and dm_bufio_read. Operation of these
948 * functions is similar except that dm_bufio_new doesn't read the
949 * buffer from the disk (assuming that the caller overwrites all the data
950 * and uses dm_bufio_mark_buffer_dirty to write new data back).
952 static void *new_read(struct dm_bufio_client
*c
, sector_t block
,
953 enum new_flag nf
, struct dm_buffer
**bp
)
959 b
= __bufio_new(c
, block
, nf
, bp
, &need_submit
);
966 submit_io(b
, READ
, b
->block
, read_endio
);
968 wait_on_bit(&b
->state
, B_READING
, do_io_schedule
, TASK_UNINTERRUPTIBLE
);
971 int error
= b
->read_error
;
975 return ERR_PTR(error
);
983 void *dm_bufio_get(struct dm_bufio_client
*c
, sector_t block
,
984 struct dm_buffer
**bp
)
986 return new_read(c
, block
, NF_GET
, bp
);
988 EXPORT_SYMBOL_GPL(dm_bufio_get
);
990 void *dm_bufio_read(struct dm_bufio_client
*c
, sector_t block
,
991 struct dm_buffer
**bp
)
993 BUG_ON(dm_bufio_in_request());
995 return new_read(c
, block
, NF_READ
, bp
);
997 EXPORT_SYMBOL_GPL(dm_bufio_read
);
999 void *dm_bufio_new(struct dm_bufio_client
*c
, sector_t block
,
1000 struct dm_buffer
**bp
)
1002 BUG_ON(dm_bufio_in_request());
1004 return new_read(c
, block
, NF_FRESH
, bp
);
1006 EXPORT_SYMBOL_GPL(dm_bufio_new
);
1008 void dm_bufio_release(struct dm_buffer
*b
)
1010 struct dm_bufio_client
*c
= b
->c
;
1014 BUG_ON(test_bit(B_READING
, &b
->state
));
1015 BUG_ON(!b
->hold_count
);
1018 if (!b
->hold_count
) {
1019 wake_up(&c
->free_buffer_wait
);
1022 * If there were errors on the buffer, and the buffer is not
1023 * to be written, free the buffer. There is no point in caching
1026 if ((b
->read_error
|| b
->write_error
) &&
1027 !test_bit(B_WRITING
, &b
->state
) &&
1028 !test_bit(B_DIRTY
, &b
->state
)) {
1030 __free_buffer_wake(b
);
1036 EXPORT_SYMBOL_GPL(dm_bufio_release
);
1038 void dm_bufio_mark_buffer_dirty(struct dm_buffer
*b
)
1040 struct dm_bufio_client
*c
= b
->c
;
1044 if (!test_and_set_bit(B_DIRTY
, &b
->state
))
1045 __relink_lru(b
, LIST_DIRTY
);
1049 EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty
);
1051 void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client
*c
)
1053 BUG_ON(dm_bufio_in_request());
1056 __write_dirty_buffers_async(c
, 0);
1059 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async
);
1062 * For performance, it is essential that the buffers are written asynchronously
1063 * and simultaneously (so that the block layer can merge the writes) and then
1066 * Finally, we flush hardware disk cache.
1068 int dm_bufio_write_dirty_buffers(struct dm_bufio_client
*c
)
1071 unsigned long buffers_processed
= 0;
1072 struct dm_buffer
*b
, *tmp
;
1075 __write_dirty_buffers_async(c
, 0);
1078 list_for_each_entry_safe_reverse(b
, tmp
, &c
->lru
[LIST_DIRTY
], lru_list
) {
1079 int dropped_lock
= 0;
1081 if (buffers_processed
< c
->n_buffers
[LIST_DIRTY
])
1082 buffers_processed
++;
1084 BUG_ON(test_bit(B_READING
, &b
->state
));
1086 if (test_bit(B_WRITING
, &b
->state
)) {
1087 if (buffers_processed
< c
->n_buffers
[LIST_DIRTY
]) {
1091 wait_on_bit(&b
->state
, B_WRITING
,
1093 TASK_UNINTERRUPTIBLE
);
1097 wait_on_bit(&b
->state
, B_WRITING
,
1099 TASK_UNINTERRUPTIBLE
);
1102 if (!test_bit(B_DIRTY
, &b
->state
) &&
1103 !test_bit(B_WRITING
, &b
->state
))
1104 __relink_lru(b
, LIST_CLEAN
);
1106 dm_bufio_cond_resched();
1109 * If we dropped the lock, the list is no longer consistent,
1110 * so we must restart the search.
1112 * In the most common case, the buffer just processed is
1113 * relinked to the clean list, so we won't loop scanning the
1114 * same buffer again and again.
1116 * This may livelock if there is another thread simultaneously
1117 * dirtying buffers, so we count the number of buffers walked
1118 * and if it exceeds the total number of buffers, it means that
1119 * someone is doing some writes simultaneously with us. In
1120 * this case, stop, dropping the lock.
1125 wake_up(&c
->free_buffer_wait
);
1128 a
= xchg(&c
->async_write_error
, 0);
1129 f
= dm_bufio_issue_flush(c
);
1135 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers
);
1138 * Use dm-io to send and empty barrier flush the device.
1140 int dm_bufio_issue_flush(struct dm_bufio_client
*c
)
1142 struct dm_io_request io_req
= {
1144 .mem
.type
= DM_IO_KMEM
,
1145 .mem
.ptr
.addr
= NULL
,
1148 struct dm_io_region io_reg
= {
1154 BUG_ON(dm_bufio_in_request());
1156 return dm_io(&io_req
, 1, &io_reg
, NULL
);
1158 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush
);
1161 * We first delete any other buffer that may be at that new location.
1163 * Then, we write the buffer to the original location if it was dirty.
1165 * Then, if we are the only one who is holding the buffer, relink the buffer
1166 * in the hash queue for the new location.
1168 * If there was someone else holding the buffer, we write it to the new
1169 * location but not relink it, because that other user needs to have the buffer
1170 * at the same place.
1172 void dm_bufio_release_move(struct dm_buffer
*b
, sector_t new_block
)
1174 struct dm_bufio_client
*c
= b
->c
;
1175 struct dm_buffer
*new;
1177 BUG_ON(dm_bufio_in_request());
1182 new = __find(c
, new_block
);
1184 if (new->hold_count
) {
1185 __wait_for_free_buffer(c
);
1190 * FIXME: Is there any point waiting for a write that's going
1191 * to be overwritten in a bit?
1193 __make_buffer_clean(new);
1194 __unlink_buffer(new);
1195 __free_buffer_wake(new);
1198 BUG_ON(!b
->hold_count
);
1199 BUG_ON(test_bit(B_READING
, &b
->state
));
1201 __write_dirty_buffer(b
);
1202 if (b
->hold_count
== 1) {
1203 wait_on_bit(&b
->state
, B_WRITING
,
1204 do_io_schedule
, TASK_UNINTERRUPTIBLE
);
1205 set_bit(B_DIRTY
, &b
->state
);
1207 __link_buffer(b
, new_block
, LIST_DIRTY
);
1210 wait_on_bit_lock(&b
->state
, B_WRITING
,
1211 do_io_schedule
, TASK_UNINTERRUPTIBLE
);
1213 * Relink buffer to "new_block" so that write_callback
1214 * sees "new_block" as a block number.
1215 * After the write, link the buffer back to old_block.
1216 * All this must be done in bufio lock, so that block number
1217 * change isn't visible to other threads.
1219 old_block
= b
->block
;
1221 __link_buffer(b
, new_block
, b
->list_mode
);
1222 submit_io(b
, WRITE
, new_block
, write_endio
);
1223 wait_on_bit(&b
->state
, B_WRITING
,
1224 do_io_schedule
, TASK_UNINTERRUPTIBLE
);
1226 __link_buffer(b
, old_block
, b
->list_mode
);
1230 dm_bufio_release(b
);
1232 EXPORT_SYMBOL_GPL(dm_bufio_release_move
);
1234 unsigned dm_bufio_get_block_size(struct dm_bufio_client
*c
)
1236 return c
->block_size
;
1238 EXPORT_SYMBOL_GPL(dm_bufio_get_block_size
);
1240 sector_t
dm_bufio_get_device_size(struct dm_bufio_client
*c
)
1242 return i_size_read(c
->bdev
->bd_inode
) >>
1243 (SECTOR_SHIFT
+ c
->sectors_per_block_bits
);
1245 EXPORT_SYMBOL_GPL(dm_bufio_get_device_size
);
1247 sector_t
dm_bufio_get_block_number(struct dm_buffer
*b
)
1251 EXPORT_SYMBOL_GPL(dm_bufio_get_block_number
);
1253 void *dm_bufio_get_block_data(struct dm_buffer
*b
)
1257 EXPORT_SYMBOL_GPL(dm_bufio_get_block_data
);
1259 void *dm_bufio_get_aux_data(struct dm_buffer
*b
)
1263 EXPORT_SYMBOL_GPL(dm_bufio_get_aux_data
);
1265 struct dm_bufio_client
*dm_bufio_get_client(struct dm_buffer
*b
)
1269 EXPORT_SYMBOL_GPL(dm_bufio_get_client
);
1271 static void drop_buffers(struct dm_bufio_client
*c
)
1273 struct dm_buffer
*b
;
1276 BUG_ON(dm_bufio_in_request());
1279 * An optimization so that the buffers are not written one-by-one.
1281 dm_bufio_write_dirty_buffers_async(c
);
1285 while ((b
= __get_unclaimed_buffer(c
)))
1286 __free_buffer_wake(b
);
1288 for (i
= 0; i
< LIST_SIZE
; i
++)
1289 list_for_each_entry(b
, &c
->lru
[i
], lru_list
)
1290 DMERR("leaked buffer %llx, hold count %u, list %d",
1291 (unsigned long long)b
->block
, b
->hold_count
, i
);
1293 for (i
= 0; i
< LIST_SIZE
; i
++)
1294 BUG_ON(!list_empty(&c
->lru
[i
]));
1300 * Test if the buffer is unused and too old, and commit it.
1301 * At if noio is set, we must not do any I/O because we hold
1302 * dm_bufio_clients_lock and we would risk deadlock if the I/O gets rerouted to
1303 * different bufio client.
1305 static int __cleanup_old_buffer(struct dm_buffer
*b
, gfp_t gfp
,
1306 unsigned long max_jiffies
)
1308 if (jiffies
- b
->last_accessed
< max_jiffies
)
1311 if (!(gfp
& __GFP_IO
)) {
1312 if (test_bit(B_READING
, &b
->state
) ||
1313 test_bit(B_WRITING
, &b
->state
) ||
1314 test_bit(B_DIRTY
, &b
->state
))
1321 __make_buffer_clean(b
);
1323 __free_buffer_wake(b
);
1328 static void __scan(struct dm_bufio_client
*c
, unsigned long nr_to_scan
,
1329 struct shrink_control
*sc
)
1332 struct dm_buffer
*b
, *tmp
;
1334 for (l
= 0; l
< LIST_SIZE
; l
++) {
1335 list_for_each_entry_safe_reverse(b
, tmp
, &c
->lru
[l
], lru_list
)
1336 if (!__cleanup_old_buffer(b
, sc
->gfp_mask
, 0) &&
1339 dm_bufio_cond_resched();
1343 static int shrink(struct shrinker
*shrinker
, struct shrink_control
*sc
)
1345 struct dm_bufio_client
*c
=
1346 container_of(shrinker
, struct dm_bufio_client
, shrinker
);
1348 unsigned long nr_to_scan
= sc
->nr_to_scan
;
1350 if (sc
->gfp_mask
& __GFP_IO
)
1352 else if (!dm_bufio_trylock(c
))
1353 return !nr_to_scan
? 0 : -1;
1356 __scan(c
, nr_to_scan
, sc
);
1358 r
= c
->n_buffers
[LIST_CLEAN
] + c
->n_buffers
[LIST_DIRTY
];
1368 * Create the buffering interface
1370 struct dm_bufio_client
*dm_bufio_client_create(struct block_device
*bdev
, unsigned block_size
,
1371 unsigned reserved_buffers
, unsigned aux_size
,
1372 void (*alloc_callback
)(struct dm_buffer
*),
1373 void (*write_callback
)(struct dm_buffer
*))
1376 struct dm_bufio_client
*c
;
1379 BUG_ON(block_size
< 1 << SECTOR_SHIFT
||
1380 (block_size
& (block_size
- 1)));
1382 c
= kmalloc(sizeof(*c
), GFP_KERNEL
);
1387 c
->cache_hash
= vmalloc(sizeof(struct hlist_head
) << DM_BUFIO_HASH_BITS
);
1388 if (!c
->cache_hash
) {
1394 c
->block_size
= block_size
;
1395 c
->sectors_per_block_bits
= ffs(block_size
) - 1 - SECTOR_SHIFT
;
1396 c
->pages_per_block_bits
= (ffs(block_size
) - 1 >= PAGE_SHIFT
) ?
1397 ffs(block_size
) - 1 - PAGE_SHIFT
: 0;
1398 c
->blocks_per_page_bits
= (ffs(block_size
) - 1 < PAGE_SHIFT
?
1399 PAGE_SHIFT
- (ffs(block_size
) - 1) : 0);
1401 c
->aux_size
= aux_size
;
1402 c
->alloc_callback
= alloc_callback
;
1403 c
->write_callback
= write_callback
;
1405 for (i
= 0; i
< LIST_SIZE
; i
++) {
1406 INIT_LIST_HEAD(&c
->lru
[i
]);
1407 c
->n_buffers
[i
] = 0;
1410 for (i
= 0; i
< 1 << DM_BUFIO_HASH_BITS
; i
++)
1411 INIT_HLIST_HEAD(&c
->cache_hash
[i
]);
1413 mutex_init(&c
->lock
);
1414 INIT_LIST_HEAD(&c
->reserved_buffers
);
1415 c
->need_reserved_buffers
= reserved_buffers
;
1417 init_waitqueue_head(&c
->free_buffer_wait
);
1418 c
->async_write_error
= 0;
1420 c
->dm_io
= dm_io_client_create();
1421 if (IS_ERR(c
->dm_io
)) {
1422 r
= PTR_ERR(c
->dm_io
);
1426 mutex_lock(&dm_bufio_clients_lock
);
1427 if (c
->blocks_per_page_bits
) {
1428 if (!DM_BUFIO_CACHE_NAME(c
)) {
1429 DM_BUFIO_CACHE_NAME(c
) = kasprintf(GFP_KERNEL
, "dm_bufio_cache-%u", c
->block_size
);
1430 if (!DM_BUFIO_CACHE_NAME(c
)) {
1432 mutex_unlock(&dm_bufio_clients_lock
);
1437 if (!DM_BUFIO_CACHE(c
)) {
1438 DM_BUFIO_CACHE(c
) = kmem_cache_create(DM_BUFIO_CACHE_NAME(c
),
1440 c
->block_size
, 0, NULL
);
1441 if (!DM_BUFIO_CACHE(c
)) {
1443 mutex_unlock(&dm_bufio_clients_lock
);
1448 mutex_unlock(&dm_bufio_clients_lock
);
1450 while (c
->need_reserved_buffers
) {
1451 struct dm_buffer
*b
= alloc_buffer(c
, GFP_KERNEL
);
1457 __free_buffer_wake(b
);
1460 mutex_lock(&dm_bufio_clients_lock
);
1461 dm_bufio_client_count
++;
1462 list_add(&c
->client_list
, &dm_bufio_all_clients
);
1463 __cache_size_refresh();
1464 mutex_unlock(&dm_bufio_clients_lock
);
1466 c
->shrinker
.shrink
= shrink
;
1467 c
->shrinker
.seeks
= 1;
1468 c
->shrinker
.batch
= 0;
1469 register_shrinker(&c
->shrinker
);
1475 while (!list_empty(&c
->reserved_buffers
)) {
1476 struct dm_buffer
*b
= list_entry(c
->reserved_buffers
.next
,
1477 struct dm_buffer
, lru_list
);
1478 list_del(&b
->lru_list
);
1481 dm_io_client_destroy(c
->dm_io
);
1483 vfree(c
->cache_hash
);
1489 EXPORT_SYMBOL_GPL(dm_bufio_client_create
);
1492 * Free the buffering interface.
1493 * It is required that there are no references on any buffers.
1495 void dm_bufio_client_destroy(struct dm_bufio_client
*c
)
1501 unregister_shrinker(&c
->shrinker
);
1503 mutex_lock(&dm_bufio_clients_lock
);
1505 list_del(&c
->client_list
);
1506 dm_bufio_client_count
--;
1507 __cache_size_refresh();
1509 mutex_unlock(&dm_bufio_clients_lock
);
1511 for (i
= 0; i
< 1 << DM_BUFIO_HASH_BITS
; i
++)
1512 BUG_ON(!hlist_empty(&c
->cache_hash
[i
]));
1514 BUG_ON(c
->need_reserved_buffers
);
1516 while (!list_empty(&c
->reserved_buffers
)) {
1517 struct dm_buffer
*b
= list_entry(c
->reserved_buffers
.next
,
1518 struct dm_buffer
, lru_list
);
1519 list_del(&b
->lru_list
);
1523 for (i
= 0; i
< LIST_SIZE
; i
++)
1524 if (c
->n_buffers
[i
])
1525 DMERR("leaked buffer count %d: %ld", i
, c
->n_buffers
[i
]);
1527 for (i
= 0; i
< LIST_SIZE
; i
++)
1528 BUG_ON(c
->n_buffers
[i
]);
1530 dm_io_client_destroy(c
->dm_io
);
1531 vfree(c
->cache_hash
);
1534 EXPORT_SYMBOL_GPL(dm_bufio_client_destroy
);
1536 static void cleanup_old_buffers(void)
1538 unsigned long max_age
= dm_bufio_max_age
;
1539 struct dm_bufio_client
*c
;
1543 if (max_age
> ULONG_MAX
/ HZ
)
1544 max_age
= ULONG_MAX
/ HZ
;
1546 mutex_lock(&dm_bufio_clients_lock
);
1547 list_for_each_entry(c
, &dm_bufio_all_clients
, client_list
) {
1548 if (!dm_bufio_trylock(c
))
1551 while (!list_empty(&c
->lru
[LIST_CLEAN
])) {
1552 struct dm_buffer
*b
;
1553 b
= list_entry(c
->lru
[LIST_CLEAN
].prev
,
1554 struct dm_buffer
, lru_list
);
1555 if (__cleanup_old_buffer(b
, 0, max_age
* HZ
))
1557 dm_bufio_cond_resched();
1561 dm_bufio_cond_resched();
1563 mutex_unlock(&dm_bufio_clients_lock
);
1566 static struct workqueue_struct
*dm_bufio_wq
;
1567 static struct delayed_work dm_bufio_work
;
1569 static void work_fn(struct work_struct
*w
)
1571 cleanup_old_buffers();
1573 queue_delayed_work(dm_bufio_wq
, &dm_bufio_work
,
1574 DM_BUFIO_WORK_TIMER_SECS
* HZ
);
1577 /*----------------------------------------------------------------
1579 *--------------------------------------------------------------*/
1582 * This is called only once for the whole dm_bufio module.
1583 * It initializes memory limit.
1585 static int __init
dm_bufio_init(void)
1589 memset(&dm_bufio_caches
, 0, sizeof dm_bufio_caches
);
1590 memset(&dm_bufio_cache_names
, 0, sizeof dm_bufio_cache_names
);
1592 mem
= (__u64
)((totalram_pages
- totalhigh_pages
) *
1593 DM_BUFIO_MEMORY_PERCENT
/ 100) << PAGE_SHIFT
;
1595 if (mem
> ULONG_MAX
)
1600 * Get the size of vmalloc space the same way as VMALLOC_TOTAL
1601 * in fs/proc/internal.h
1603 if (mem
> (VMALLOC_END
- VMALLOC_START
) * DM_BUFIO_VMALLOC_PERCENT
/ 100)
1604 mem
= (VMALLOC_END
- VMALLOC_START
) * DM_BUFIO_VMALLOC_PERCENT
/ 100;
1607 dm_bufio_default_cache_size
= mem
;
1609 mutex_lock(&dm_bufio_clients_lock
);
1610 __cache_size_refresh();
1611 mutex_unlock(&dm_bufio_clients_lock
);
1613 dm_bufio_wq
= create_singlethread_workqueue("dm_bufio_cache");
1617 INIT_DELAYED_WORK(&dm_bufio_work
, work_fn
);
1618 queue_delayed_work(dm_bufio_wq
, &dm_bufio_work
,
1619 DM_BUFIO_WORK_TIMER_SECS
* HZ
);
1625 * This is called once when unloading the dm_bufio module.
1627 static void __exit
dm_bufio_exit(void)
1632 cancel_delayed_work_sync(&dm_bufio_work
);
1633 destroy_workqueue(dm_bufio_wq
);
1635 for (i
= 0; i
< ARRAY_SIZE(dm_bufio_caches
); i
++) {
1636 struct kmem_cache
*kc
= dm_bufio_caches
[i
];
1639 kmem_cache_destroy(kc
);
1642 for (i
= 0; i
< ARRAY_SIZE(dm_bufio_cache_names
); i
++)
1643 kfree(dm_bufio_cache_names
[i
]);
1645 if (dm_bufio_client_count
) {
1646 DMCRIT("%s: dm_bufio_client_count leaked: %d",
1647 __func__
, dm_bufio_client_count
);
1651 if (dm_bufio_current_allocated
) {
1652 DMCRIT("%s: dm_bufio_current_allocated leaked: %lu",
1653 __func__
, dm_bufio_current_allocated
);
1657 if (dm_bufio_allocated_get_free_pages
) {
1658 DMCRIT("%s: dm_bufio_allocated_get_free_pages leaked: %lu",
1659 __func__
, dm_bufio_allocated_get_free_pages
);
1663 if (dm_bufio_allocated_vmalloc
) {
1664 DMCRIT("%s: dm_bufio_vmalloc leaked: %lu",
1665 __func__
, dm_bufio_allocated_vmalloc
);
1673 module_init(dm_bufio_init
)
1674 module_exit(dm_bufio_exit
)
1676 module_param_named(max_cache_size_bytes
, dm_bufio_cache_size
, ulong
, S_IRUGO
| S_IWUSR
);
1677 MODULE_PARM_DESC(max_cache_size_bytes
, "Size of metadata cache");
1679 module_param_named(max_age_seconds
, dm_bufio_max_age
, uint
, S_IRUGO
| S_IWUSR
);
1680 MODULE_PARM_DESC(max_age_seconds
, "Max age of a buffer in seconds");
1682 module_param_named(peak_allocated_bytes
, dm_bufio_peak_allocated
, ulong
, S_IRUGO
| S_IWUSR
);
1683 MODULE_PARM_DESC(peak_allocated_bytes
, "Tracks the maximum allocated memory");
1685 module_param_named(allocated_kmem_cache_bytes
, dm_bufio_allocated_kmem_cache
, ulong
, S_IRUGO
);
1686 MODULE_PARM_DESC(allocated_kmem_cache_bytes
, "Memory allocated with kmem_cache_alloc");
1688 module_param_named(allocated_get_free_pages_bytes
, dm_bufio_allocated_get_free_pages
, ulong
, S_IRUGO
);
1689 MODULE_PARM_DESC(allocated_get_free_pages_bytes
, "Memory allocated with get_free_pages");
1691 module_param_named(allocated_vmalloc_bytes
, dm_bufio_allocated_vmalloc
, ulong
, S_IRUGO
);
1692 MODULE_PARM_DESC(allocated_vmalloc_bytes
, "Memory allocated with vmalloc");
1694 module_param_named(current_allocated_bytes
, dm_bufio_current_allocated
, ulong
, S_IRUGO
);
1695 MODULE_PARM_DESC(current_allocated_bytes
, "Memory currently used by the cache");
1697 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
1698 MODULE_DESCRIPTION(DM_NAME
" buffered I/O library");
1699 MODULE_LICENSE("GPL");