[deliverable/linux.git] / drivers / md / dm-bio-prison.c

/*
 * Copyright (C) 2012 Red Hat, Inc.
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-prison.h"

#include <linux/spinlock.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/slab.h>

/*----------------------------------------------------------------*/

struct dm_bio_prison {
	spinlock_t lock;
	mempool_t *cell_pool;

	unsigned nr_buckets;
	unsigned hash_mask;
	struct hlist_head *cells;
};

/*----------------------------------------------------------------*/

static uint32_t calc_nr_buckets(unsigned nr_cells)
{
	uint32_t n = 128;

	nr_cells /= 4;
	nr_cells = min(nr_cells, 8192u);

	while (n < nr_cells)
		n <<= 1;

	return n;
}

static struct kmem_cache *_cell_cache;

/*
 * @nr_cells should be the number of cells you want in use _concurrently_.
 * Don't confuse it with the number of distinct keys.
 */
struct dm_bio_prison *dm_bio_prison_create(unsigned nr_cells)
{
	unsigned i;
	uint32_t nr_buckets = calc_nr_buckets(nr_cells);
	size_t len = sizeof(struct dm_bio_prison) +
		(sizeof(struct hlist_head) * nr_buckets);
	struct dm_bio_prison *prison = kmalloc(len, GFP_KERNEL);

	if (!prison)
		return NULL;

	spin_lock_init(&prison->lock);
	prison->cell_pool = mempool_create_slab_pool(nr_cells, _cell_cache);
	if (!prison->cell_pool) {
		kfree(prison);
		return NULL;
	}

	prison->nr_buckets = nr_buckets;
	prison->hash_mask = nr_buckets - 1;
	prison->cells = (struct hlist_head *) (prison + 1);
	for (i = 0; i < nr_buckets; i++)
		INIT_HLIST_HEAD(prison->cells + i);

	return prison;
}
EXPORT_SYMBOL_GPL(dm_bio_prison_create);

void dm_bio_prison_destroy(struct dm_bio_prison *prison)
{
	mempool_destroy(prison->cell_pool);
	kfree(prison);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);

struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
{
	return mempool_alloc(prison->cell_pool, gfp);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);

void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
			     struct dm_bio_prison_cell *cell)
{
	mempool_free(cell, prison->cell_pool);
}
EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);

static uint32_t hash_key(struct dm_bio_prison *prison, struct dm_cell_key *key)
{
	const unsigned long BIG_PRIME = 4294967291UL;
	uint64_t hash = key->block * BIG_PRIME;

	return (uint32_t) (hash & prison->hash_mask);
}

static int keys_equal(struct dm_cell_key *lhs, struct dm_cell_key *rhs)
{
	       return (lhs->virtual == rhs->virtual) &&
		       (lhs->dev == rhs->dev) &&
		       (lhs->block == rhs->block);
}

static struct dm_bio_prison_cell *__search_bucket(struct hlist_head *bucket,
						  struct dm_cell_key *key)
{
	struct dm_bio_prison_cell *cell;

	hlist_for_each_entry(cell, bucket, list)
		if (keys_equal(&cell->key, key))
			return cell;

	return NULL;
}

static void __setup_new_cell(struct dm_bio_prison *prison,
			     struct dm_cell_key *key,
			     struct bio *holder,
			     uint32_t hash,
			     struct dm_bio_prison_cell *cell)
{
	memcpy(&cell->key, key, sizeof(cell->key));
	cell->holder = holder;
	bio_list_init(&cell->bios);
	hlist_add_head(&cell->list, prison->cells + hash);
}

static int __bio_detain(struct dm_bio_prison *prison,
			struct dm_cell_key *key,
			struct bio *inmate,
			struct dm_bio_prison_cell *cell_prealloc,
			struct dm_bio_prison_cell **cell_result)
{
	uint32_t hash = hash_key(prison, key);
	struct dm_bio_prison_cell *cell;

	cell = __search_bucket(prison->cells + hash, key);
	if (cell) {
		if (inmate)
			bio_list_add(&cell->bios, inmate);
		*cell_result = cell;
		return 1;
	}

	__setup_new_cell(prison, key, inmate, hash, cell_prealloc);
	*cell_result = cell_prealloc;
	return 0;
}

static int bio_detain(struct dm_bio_prison *prison,
		      struct dm_cell_key *key,
		      struct bio *inmate,
		      struct dm_bio_prison_cell *cell_prealloc,
		      struct dm_bio_prison_cell **cell_result)
{
	int r;
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	spin_unlock_irqrestore(&prison->lock, flags);

	return r;
}

int dm_bio_detain(struct dm_bio_prison *prison,
		  struct dm_cell_key *key,
		  struct bio *inmate,
		  struct dm_bio_prison_cell *cell_prealloc,
		  struct dm_bio_prison_cell **cell_result)
{
	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
}
EXPORT_SYMBOL_GPL(dm_bio_detain);

/*
 * @inmates must have been initialised prior to this call
 */
static void __cell_release(struct dm_bio_prison_cell *cell,
			   struct bio_list *inmates)
{
	hlist_del(&cell->list);

	if (inmates) {
		if (cell->holder)
			bio_list_add(inmates, cell->holder);
		bio_list_merge(inmates, &cell->bios);
	}
}

void dm_cell_release(struct dm_bio_prison *prison,
		     struct dm_bio_prison_cell *cell,
		     struct bio_list *bios)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release(cell, bios);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_release);

/*
 * Sometimes we don't want the holder, just the additional bios.
 */
static void __cell_release_no_holder(struct dm_bio_prison_cell *cell,
				     struct bio_list *inmates)
{
	hlist_del(&cell->list);
	bio_list_merge(inmates, &cell->bios);
}

void dm_cell_release_no_holder(struct dm_bio_prison *prison,
			       struct dm_bio_prison_cell *cell,
			       struct bio_list *inmates)
{
	unsigned long flags;

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release_no_holder(cell, inmates);
	spin_unlock_irqrestore(&prison->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);

void dm_cell_error(struct dm_bio_prison *prison,
		   struct dm_bio_prison_cell *cell)
{
	struct bio_list bios;
	struct bio *bio;
	unsigned long flags;

	bio_list_init(&bios);

	spin_lock_irqsave(&prison->lock, flags);
	__cell_release(cell, &bios);
	spin_unlock_irqrestore(&prison->lock, flags);

	while ((bio = bio_list_pop(&bios)))
		bio_io_error(bio);
}
EXPORT_SYMBOL_GPL(dm_cell_error);

/*----------------------------------------------------------------*/

#define DEFERRED_SET_SIZE 64

struct dm_deferred_entry {
	struct dm_deferred_set *ds;
	unsigned count;
	struct list_head work_items;
};

struct dm_deferred_set {
	spinlock_t lock;
	unsigned current_entry;
	unsigned sweeper;
	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
};

struct dm_deferred_set *dm_deferred_set_create(void)
{
	int i;
	struct dm_deferred_set *ds;

	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	if (!ds)
		return NULL;

	spin_lock_init(&ds->lock);
	ds->current_entry = 0;
	ds->sweeper = 0;
	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
		ds->entries[i].ds = ds;
		ds->entries[i].count = 0;
		INIT_LIST_HEAD(&ds->entries[i].work_items);
	}

	return ds;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_create);

void dm_deferred_set_destroy(struct dm_deferred_set *ds)
{
	kfree(ds);
}
EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);

struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
{
	unsigned long flags;
	struct dm_deferred_entry *entry;

	spin_lock_irqsave(&ds->lock, flags);
	entry = ds->entries + ds->current_entry;
	entry->count++;
	spin_unlock_irqrestore(&ds->lock, flags);

	return entry;
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);

static unsigned ds_next(unsigned index)
{
	return (index + 1) % DEFERRED_SET_SIZE;
}

static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
{
	while ((ds->sweeper != ds->current_entry) &&
	       !ds->entries[ds->sweeper].count) {
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
		ds->sweeper = ds_next(ds->sweeper);
	}

	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
		list_splice_init(&ds->entries[ds->sweeper].work_items, head);
}

void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
{
	unsigned long flags;

	spin_lock_irqsave(&entry->ds->lock, flags);
	BUG_ON(!entry->count);
	--entry->count;
	__sweep(entry->ds, head);
	spin_unlock_irqrestore(&entry->ds->lock, flags);
}
EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);

/*
 * Returns 1 if deferred or 0 if no pending items to delay job.
 */
int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
{
	int r = 1;
	unsigned long flags;
	unsigned next_entry;

	spin_lock_irqsave(&ds->lock, flags);
	if ((ds->sweeper == ds->current_entry) &&
	    !ds->entries[ds->current_entry].count)
		r = 0;
	else {
		list_add(work, &ds->entries[ds->current_entry].work_items);
		next_entry = ds_next(ds->current_entry);
		if (!ds->entries[next_entry].count)
			ds->current_entry = next_entry;
	}
	spin_unlock_irqrestore(&ds->lock, flags);

	return r;
}
EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);

/*----------------------------------------------------------------*/

static int __init dm_bio_prison_init(void)
{
	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
	if (!_cell_cache)
		return -ENOMEM;

	return 0;
}

static void __exit dm_bio_prison_exit(void)
{
	kmem_cache_destroy(_cell_cache);
	_cell_cache = NULL;
}

/*
 * module hooks
 */
module_init(dm_bio_prison_init);
module_exit(dm_bio_prison_exit);

MODULE_DESCRIPTION(DM_NAME " bio prison");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");
Commit	Line	Data
4f81a417 MS	1	/*
	2	* Copyright (C) 2012 Red Hat, Inc.
	3	*
	4	* This file is released under the GPL.
	5	*/
	6
	7	#include "dm.h"
	8	#include "dm-bio-prison.h"
	9
	10	#include <linux/spinlock.h>
	11	#include <linux/mempool.h>
	12	#include <linux/module.h>
	13	#include <linux/slab.h>
	14
	15	/----------------------------------------------------------------/
	16
4f81a417 MS	17	struct dm_bio_prison {
	18	spinlock_t lock;
	19	mempool_t *cell_pool;
	20
	21	unsigned nr_buckets;
	22	unsigned hash_mask;
	23	struct hlist_head *cells;
	24	};
	25
	26	/----------------------------------------------------------------/
	27
	28	static uint32_t calc_nr_buckets(unsigned nr_cells)
	29	{
	30	uint32_t n = 128;
	31
	32	nr_cells /= 4;
	33	nr_cells = min(nr_cells, 8192u);
	34
	35	while (n < nr_cells)
	36	n <<= 1;
	37
	38	return n;
	39	}
	40
	41	static struct kmem_cache *_cell_cache;
	42
	43	/*
	44	* @nr_cells should be the number of cells you want in use _concurrently_.
	45	* Don't confuse it with the number of distinct keys.
	46	*/
	47	struct dm_bio_prison *dm_bio_prison_create(unsigned nr_cells)
	48	{
	49	unsigned i;
	50	uint32_t nr_buckets = calc_nr_buckets(nr_cells);
	51	size_t len = sizeof(struct dm_bio_prison) +
	52	(sizeof(struct hlist_head) * nr_buckets);
	53	struct dm_bio_prison *prison = kmalloc(len, GFP_KERNEL);
	54
	55	if (!prison)
	56	return NULL;
	57
	58	spin_lock_init(&prison->lock);
	59	prison->cell_pool = mempool_create_slab_pool(nr_cells, _cell_cache);
	60	if (!prison->cell_pool) {
	61	kfree(prison);
	62	return NULL;
	63	}
	64
	65	prison->nr_buckets = nr_buckets;
	66	prison->hash_mask = nr_buckets - 1;
	67	prison->cells = (struct hlist_head *) (prison + 1);
	68	for (i = 0; i < nr_buckets; i++)
	69	INIT_HLIST_HEAD(prison->cells + i);
	70
	71	return prison;
	72	}
	73	EXPORT_SYMBOL_GPL(dm_bio_prison_create);
	74
	75	void dm_bio_prison_destroy(struct dm_bio_prison *prison)
	76	{
	77	mempool_destroy(prison->cell_pool);
	78	kfree(prison);
	79	}
	80	EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
81
6beca5eb JT	82	struct dm_bio_prison_cell dm_bio_prison_alloc_cell(struct dm_bio_prison prison, gfp_t gfp)
	83	{
	84	return mempool_alloc(prison->cell_pool, gfp);
	85	}
	86	EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
	87
	88	void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
	89	struct dm_bio_prison_cell *cell)
	90	{
	91	mempool_free(cell, prison->cell_pool);
	92	}
	93	EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
	94
4f81a417 MS	95	static uint32_t hash_key(struct dm_bio_prison prison, struct dm_cell_key key)
	96	{
	97	const unsigned long BIG_PRIME = 4294967291UL;
	98	uint64_t hash = key->block * BIG_PRIME;
	99
	100	return (uint32_t) (hash & prison->hash_mask);
	101	}
	102
	103	static int keys_equal(struct dm_cell_key lhs, struct dm_cell_key rhs)
	104	{
	105	return (lhs->virtual == rhs->virtual) &&
	106	(lhs->dev == rhs->dev) &&
	107	(lhs->block == rhs->block);
	108	}
	109
	110	static struct dm_bio_prison_cell __search_bucket(struct hlist_head bucket,
	111	struct dm_cell_key *key)
	112	{
	113	struct dm_bio_prison_cell *cell;
4f81a417	114
b67bfe0d	115	hlist_for_each_entry(cell, bucket, list)
4f81a417 MS	116	if (keys_equal(&cell->key, key))
	117	return cell;
	118
	119	return NULL;
	120	}
	121
6beca5eb JT	122	static void __setup_new_cell(struct dm_bio_prison *prison,
	123	struct dm_cell_key *key,
	124	struct bio *holder,
	125	uint32_t hash,
	126	struct dm_bio_prison_cell *cell)
4f81a417	127	{
6beca5eb JT	128	memcpy(&cell->key, key, sizeof(cell->key));
	129	cell->holder = holder;
	130	bio_list_init(&cell->bios);
	131	hlist_add_head(&cell->list, prison->cells + hash);
	132	}
4f81a417	133
6beca5eb JT	134	static int __bio_detain(struct dm_bio_prison *prison,
	135	struct dm_cell_key *key,
	136	struct bio *inmate,
	137	struct dm_bio_prison_cell *cell_prealloc,
	138	struct dm_bio_prison_cell **cell_result)
	139	{
	140	uint32_t hash = hash_key(prison, key);
	141	struct dm_bio_prison_cell *cell;
4f81a417	142
4f81a417 MS	143	cell = __search_bucket(prison->cells + hash, key);
4f81a417 MS	144	if (cell) {
6beca5eb JT	145	if (inmate)
	146	bio_list_add(&cell->bios, inmate);
	147	*cell_result = cell;
	148	return 1;
4f81a417 MS	149	}
4f81a417 MS	150
6beca5eb JT	151	__setup_new_cell(prison, key, inmate, hash, cell_prealloc);
	152	*cell_result = cell_prealloc;
	153	return 0;
	154	}
4f81a417	155
6beca5eb JT	156	static int bio_detain(struct dm_bio_prison *prison,
	157	struct dm_cell_key *key,
	158	struct bio *inmate,
	159	struct dm_bio_prison_cell *cell_prealloc,
	160	struct dm_bio_prison_cell **cell_result)
	161	{
	162	int r;
	163	unsigned long flags;
4f81a417	164
6beca5eb JT	165	spin_lock_irqsave(&prison->lock, flags);
6beca5eb JT	166	r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
4f81a417 MS	167	spin_unlock_irqrestore(&prison->lock, flags);
4f81a417 MS	168
4f81a417 MS	169	return r;
4f81a417 MS	170	}
6beca5eb JT	171
	172	int dm_bio_detain(struct dm_bio_prison *prison,
	173	struct dm_cell_key *key,
	174	struct bio *inmate,
	175	struct dm_bio_prison_cell *cell_prealloc,
	176	struct dm_bio_prison_cell **cell_result)
	177	{
	178	return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	179	}
4f81a417 MS	180	EXPORT_SYMBOL_GPL(dm_bio_detain);
	181
	182	/*
	183	* @inmates must have been initialised prior to this call
	184	*/
6beca5eb JT	185	static void __cell_release(struct dm_bio_prison_cell *cell,
6beca5eb JT	186	struct bio_list *inmates)
4f81a417	187	{
4f81a417 MS	188	hlist_del(&cell->list);
	189
	190	if (inmates) {
6beca5eb JT	191	if (cell->holder)
6beca5eb JT	192	bio_list_add(inmates, cell->holder);
4f81a417 MS	193	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	194	}
4f81a417 MS	195	}
4f81a417 MS	196
6beca5eb JT	197	void dm_cell_release(struct dm_bio_prison *prison,
	198	struct dm_bio_prison_cell *cell,
	199	struct bio_list *bios)
4f81a417 MS	200	{
4f81a417 MS	201	unsigned long flags;
4f81a417 MS	202
	203	spin_lock_irqsave(&prison->lock, flags);
	204	__cell_release(cell, bios);
	205	spin_unlock_irqrestore(&prison->lock, flags);
	206	}
	207	EXPORT_SYMBOL_GPL(dm_cell_release);
	208
4f81a417 MS	209	/*
	210	* Sometimes we don't want the holder, just the additional bios.
	211	*/
6beca5eb JT	212	static void __cell_release_no_holder(struct dm_bio_prison_cell *cell,
6beca5eb JT	213	struct bio_list *inmates)
4f81a417	214	{
4f81a417 MS	215	hlist_del(&cell->list);
4f81a417 MS	216	bio_list_merge(inmates, &cell->bios);
4f81a417 MS	217	}
4f81a417 MS	218
6beca5eb JT	219	void dm_cell_release_no_holder(struct dm_bio_prison *prison,
	220	struct dm_bio_prison_cell *cell,
	221	struct bio_list *inmates)
4f81a417 MS	222	{
4f81a417 MS	223	unsigned long flags;
4f81a417 MS	224
	225	spin_lock_irqsave(&prison->lock, flags);
	226	__cell_release_no_holder(cell, inmates);
	227	spin_unlock_irqrestore(&prison->lock, flags);
	228	}
	229	EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
	230
6beca5eb JT	231	void dm_cell_error(struct dm_bio_prison *prison,
6beca5eb JT	232	struct dm_bio_prison_cell *cell)
4f81a417	233	{
4f81a417 MS	234	struct bio_list bios;
	235	struct bio *bio;
	236	unsigned long flags;
	237
	238	bio_list_init(&bios);
	239
	240	spin_lock_irqsave(&prison->lock, flags);
	241	__cell_release(cell, &bios);
	242	spin_unlock_irqrestore(&prison->lock, flags);
	243
	244	while ((bio = bio_list_pop(&bios)))
	245	bio_io_error(bio);
	246	}
	247	EXPORT_SYMBOL_GPL(dm_cell_error);
	248
	249	/----------------------------------------------------------------/
	250
	251	#define DEFERRED_SET_SIZE 64
	252
	253	struct dm_deferred_entry {
	254	struct dm_deferred_set *ds;
	255	unsigned count;
	256	struct list_head work_items;
	257	};
	258
	259	struct dm_deferred_set {
	260	spinlock_t lock;
	261	unsigned current_entry;
	262	unsigned sweeper;
	263	struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
	264	};
	265
	266	struct dm_deferred_set *dm_deferred_set_create(void)
	267	{
	268	int i;
	269	struct dm_deferred_set *ds;
	270
	271	ds = kmalloc(sizeof(*ds), GFP_KERNEL);
	272	if (!ds)
	273	return NULL;
	274
	275	spin_lock_init(&ds->lock);
	276	ds->current_entry = 0;
	277	ds->sweeper = 0;
	278	for (i = 0; i < DEFERRED_SET_SIZE; i++) {
	279	ds->entries[i].ds = ds;
	280	ds->entries[i].count = 0;
	281	INIT_LIST_HEAD(&ds->entries[i].work_items);
	282	}
	283
	284	return ds;
	285	}
	286	EXPORT_SYMBOL_GPL(dm_deferred_set_create);
	287
	288	void dm_deferred_set_destroy(struct dm_deferred_set *ds)
	289	{
	290	kfree(ds);
	291	}
	292	EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
	293
	294	struct dm_deferred_entry dm_deferred_entry_inc(struct dm_deferred_set ds)
	295	{
	296	unsigned long flags;
	297	struct dm_deferred_entry *entry;
298
299	spin_lock_irqsave(&ds->lock, flags);
300	entry = ds->entries + ds->current_entry;
301	entry->count++;
302	spin_unlock_irqrestore(&ds->lock, flags);
303
304	return entry;
305	}
306	EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
307
308	static unsigned ds_next(unsigned index)
309	{
310	return (index + 1) % DEFERRED_SET_SIZE;
311	}
312
313	static void __sweep(struct dm_deferred_set ds, struct list_head head)
314	{
315	while ((ds->sweeper != ds->current_entry) &&
316	!ds->entries[ds->sweeper].count) {
317	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
318	ds->sweeper = ds_next(ds->sweeper);
319	}
320
321	if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
322	list_splice_init(&ds->entries[ds->sweeper].work_items, head);
323	}
324
325	void dm_deferred_entry_dec(struct dm_deferred_entry entry, struct list_head head)
326	{
327	unsigned long flags;
328
329	spin_lock_irqsave(&entry->ds->lock, flags);
330	BUG_ON(!entry->count);
331	--entry->count;
332	__sweep(entry->ds, head);
333	spin_unlock_irqrestore(&entry->ds->lock, flags);
334	}
335	EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
336
337	/*
338	* Returns 1 if deferred or 0 if no pending items to delay job.
339	*/
340	int dm_deferred_set_add_work(struct dm_deferred_set ds, struct list_head work)
341	{
342	int r = 1;
343	unsigned long flags;
344	unsigned next_entry;
345
346	spin_lock_irqsave(&ds->lock, flags);
347	if ((ds->sweeper == ds->current_entry) &&
348	!ds->entries[ds->current_entry].count)
349	r = 0;
350	else {
351	list_add(work, &ds->entries[ds->current_entry].work_items);
352	next_entry = ds_next(ds->current_entry);
353	if (!ds->entries[next_entry].count)
354	ds->current_entry = next_entry;
355	}
356	spin_unlock_irqrestore(&ds->lock, flags);
357
358	return r;
359	}
360	EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
361
362	/----------------------------------------------------------------/
363
364	static int __init dm_bio_prison_init(void)
365	{
366	_cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
367	if (!_cell_cache)
368	return -ENOMEM;
369
370	return 0;
371	}
372
373	static void __exit dm_bio_prison_exit(void)
374	{
375	kmem_cache_destroy(_cell_cache);
376	_cell_cache = NULL;
377	}
378
379	/*
380	* module hooks
381	*/
382	module_init(dm_bio_prison_init);
383	module_exit(dm_bio_prison_exit);
384
385	MODULE_DESCRIPTION(DM_NAME " bio prison");
386	MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
387	MODULE_LICENSE("GPL");